SATD

satd id satd instance id project committer name Commit Hash old comment New Comment resolution Method SignatureMethod DeclarationMethod Body
6108 -1009651754apache/hadoopArun Murthy24676e8c2ef48e67265d33c053badb31aecad616 None Mix the framework group implementation into the Group interface SATD_ADDED downgrade(org.apache.hadoop.mapreduce.Counters) static Counters downgrade(org.apache.hadoop.mapreduce.Counters newCounters) return new Counters(newCounters);
6107 -1009651755apache/hadoopArun Murthy24676e8c2ef48e67265d33c053badb31aecad616 None Mix the generic group implementation into the Group interface SATD_ADDED downgrade(org.apache.hadoop.mapreduce.Counters) static Counters downgrade(org.apache.hadoop.mapreduce.Counters newCounters) return new Counters(newCounters);
6106 -1009651756apache/hadoopArun Murthy24676e8c2ef48e67265d33c053badb31aecad616 move over ')' move over ')' SATD_MOVED_FILE getBlock(String, char, char, IntWritable) private static String getBlock(String str, char open, char close, IntWritable index) throws ParseException StringBuilder split = new StringBuilder(); int next = StringUtils.findNext(str, open, StringUtils.ESCAPE_CHAR, index.get(), split); // clear the buffer split.setLength(0); if (next >= 0) { // move over '(' ++next; next = StringUtils.findNext(str, close, StringUtils.ESCAPE_CHAR, next, split); if (next >= 0) { // move over ')' ++next; index.set(next); // found a block return split.toString(); } else { throw new ParseException("Unexpected end of block", next); } } // found nothing return null;
6105 -1009651757apache/hadoopArun Murthy24676e8c2ef48e67265d33c053badb31aecad616 move over '(' move over '(' SATD_MOVED_FILE getBlock(String, char, char, IntWritable) private static String getBlock(String str, char open, char close, IntWritable index) throws ParseException StringBuilder split = new StringBuilder(); int next = StringUtils.findNext(str, open, StringUtils.ESCAPE_CHAR, index.get(), split); // clear the buffer split.setLength(0); if (next >= 0) { // move over '(' ++next; next = StringUtils.findNext(str, close, StringUtils.ESCAPE_CHAR, next, split); if (next >= 0) { // move over ')' ++next; index.set(next); // found a block return split.toString(); } else { throw new ParseException("Unexpected end of block", next); } } // found nothing return null;
6099 -1009651759apache/hadoopSuresh Srinivas73f2092b7351b9cc13d0e12bc4ade6c470934ee8 None TODO:HA SATD_ADDED exitState(NameNode) protected void exitState(NameNode nn) throws ServiceFailedException // TODO:HA
6098 -1009651760apache/hadoopSuresh Srinivas73f2092b7351b9cc13d0e12bc4ade6c470934ee8 None TODO:HA SATD_ADDED enterState(NameNode) protected void enterState(NameNode nn) throws ServiceFailedException // TODO:HA
6097 -1009651761apache/hadoopSuresh Srinivas73f2092b7351b9cc13d0e12bc4ade6c470934ee8 None TODO:HA configuration changes pending SATD_ADDED isHAEnabled(Configuration) public static boolean isHAEnabled(Configuration conf) // TODO:HA configuration changes pending return false;
6096 -1009651762apache/hadoopSuresh Srinivas73f2092b7351b9cc13d0e12bc4ade6c470934ee8 None TODO:HA implement health check SATD_ADDED monitorHealth() public synchronized void monitorHealth() throws HealthCheckFailedException if (!haEnabled) { // no-op, if HA is not eanbled return; } // TODO:HA implement health check return;
6095 -1009651763apache/hadoopSuresh Srinivas73f2092b7351b9cc13d0e12bc4ade6c470934ee8 None TODO:HA decide on OperationCategory for this SATD_ADDED setBalancerBandwidth(long) public void setBalancerBandwidth(long bandwidth) throws IOException // TODO:HA decide on OperationCategory for this namesystem.setBalancerBandwidth(bandwidth);
6094 -1009651764apache/hadoopSuresh Srinivas73f2092b7351b9cc13d0e12bc4ade6c470934ee8 None TODO:HA decide on OperationCategory for this SATD_ADDED metaSave(String) public void metaSave(String filename) throws IOException // TODO:HA decide on OperationCategory for this namesystem.metaSave(filename);
6093 -1009651765apache/hadoopSuresh Srinivas73f2092b7351b9cc13d0e12bc4ade6c470934ee8 None TODO:HA decide on OperationCategory for this SATD_ADDED distributedUpgradeProgress(UpgradeAction) public UpgradeStatusReport distributedUpgradeProgress(UpgradeAction action) throws IOException // TODO:HA decide on OperationCategory for this return namesystem.distributedUpgradeProgress(action);
6092 -1009651766apache/hadoopSuresh Srinivas73f2092b7351b9cc13d0e12bc4ade6c470934ee8 None TODO:HA decide on OperationCategory for this SATD_ADDED finalizeUpgrade() public void finalizeUpgrade() throws IOException // TODO:HA decide on OperationCategory for this namesystem.finalizeUpgrade();
6091 -1009651767apache/hadoopSuresh Srinivas73f2092b7351b9cc13d0e12bc4ade6c470934ee8 None TODO:HA decide on OperationCategory for this SATD_ADDED getEditLogManifest(long) public RemoteEditLogManifest getEditLogManifest(long sinceTxId) throws IOException // TODO:HA decide on OperationCategory for this return namesystem.getEditLogManifest(sinceTxId);
6090 -1009651768apache/hadoopSuresh Srinivas73f2092b7351b9cc13d0e12bc4ade6c470934ee8 None TODO:HA decide on OperationCategory for this SATD_ADDED rollEditLog() public CheckpointSignature rollEditLog() throws IOException // TODO:HA decide on OperationCategory for this return namesystem.rollEditLog();
6089 -1009651769apache/hadoopSuresh Srinivas73f2092b7351b9cc13d0e12bc4ade6c470934ee8 None TODO:HA decide on OperationCategory for this SATD_ADDED getTransactionID() public long getTransactionID() // TODO:HA decide on OperationCategory for this return namesystem.getTransactionID();
6088 -1009651770apache/hadoopSuresh Srinivas73f2092b7351b9cc13d0e12bc4ade6c470934ee8 None TODO:HA decide on OperationCategory for this SATD_ADDED refreshNodes() public void refreshNodes() throws IOException // TODO:HA decide on OperationCategory for this namesystem.refreshNodes(new HdfsConfiguration());
6087 -1009651771apache/hadoopSuresh Srinivas73f2092b7351b9cc13d0e12bc4ade6c470934ee8 None TODO:HA decide on OperationCategory for this SATD_ADDED saveNamespace() public void saveNamespace() throws IOException // TODO:HA decide on OperationCategory for this namesystem.saveNamespace();
6086 -1009651772apache/hadoopSuresh Srinivas73f2092b7351b9cc13d0e12bc4ade6c470934ee8 None TODO:HA decide on OperationCategory for this SATD_ADDED restoreFailedStorage(String) public boolean restoreFailedStorage(String arg) throws AccessControlException // TODO:HA decide on OperationCategory for this return namesystem.restoreFailedStorage(arg);
6085 -1009651773apache/hadoopSuresh Srinivas73f2092b7351b9cc13d0e12bc4ade6c470934ee8 None TODO:HA decide on OperationCategory for this SATD_ADDED setSafeMode(SafeModeAction) public boolean setSafeMode(SafeModeAction action) throws IOException // TODO:HA decide on OperationCategory for this return namesystem.setSafeMode(action);
6084 -1009651774apache/hadoopSuresh Srinivas73f2092b7351b9cc13d0e12bc4ade6c470934ee8 None TODO:HA SATD_ADDED exitState(NameNode) protected void exitState(NameNode nn) throws ServiceFailedException // TODO:HA
6083 -1009651775apache/hadoopSuresh Srinivas73f2092b7351b9cc13d0e12bc4ade6c470934ee8 None TODO:HA SATD_ADDED enterState(NameNode) protected void enterState(NameNode nn) throws ServiceFailedException // TODO:HA
6082 -1009651776apache/hadoopThomas White371f4a59059322000a40eb4bdf5386b96b626ece Initial block reports can be processed a lot more efficiently than ordinary block reports. This shortens NN restart times. Log the block report processing stats from Namenode perspective SATD_REMOVED getBlockTokenSecretManager() public BlockTokenSecretManager getBlockTokenSecretManager() return blockTokenSecretManager;
6081 -1009651777apache/hadoopThomas White371f4a59059322000a40eb4bdf5386b96b626ece Initial block reports can be processed a lot more efficiently than ordinary block reports. This shortens NN restart times. The first block report can be processed a lot more efficiently than ordinary block reports. This shortens restart times. SATD_REMOVED getBlockTokenSecretManager() public BlockTokenSecretManager getBlockTokenSecretManager() return blockTokenSecretManager;
6080 -1009651778apache/hadoopThomas White371f4a59059322000a40eb4bdf5386b96b626ece Initial block reports can be processed a lot more efficiently than ordinary block reports. This shortens NN restart times. To minimize startup time, we discard any second (or later) block reports that we receive while still in startup phase. SATD_REMOVED getBlockTokenSecretManager() public BlockTokenSecretManager getBlockTokenSecretManager() return blockTokenSecretManager;
6079 -1009651779apache/hadoopThomas White371f4a59059322000a40eb4bdf5386b96b626ece Initial block reports can be processed a lot more efficiently than ordinary block reports. This shortens NN restart times. after acquiring write lock SATD_REMOVED getBlockTokenSecretManager() public BlockTokenSecretManager getBlockTokenSecretManager() return blockTokenSecretManager;
6078 -1009651788apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 move to current trash move to current trash FILE_PATH_CHANGED initialize(Configuration, FileSystem, Path) public void initialize(Configuration conf, FileSystem fs, Path home) this.fs = fs; this.trash = new Path(home, TRASH); this.homesParent = home.getParent(); this.current = new Path(trash, CURRENT); this.deletionInterval = (long) (conf.getFloat(FS_TRASH_INTERVAL_KEY, FS_TRASH_INTERVAL_DEFAULT) * MSECS_PER_MINUTE);
6077 -1009652115apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 TODO: update after fixing HADOOP-7352 TODO: update after fixing HADOOP-7352 FILE_PATH_CHANGED testListStatusThrowsExceptionForUnreadableDir() public void testListStatusThrowsExceptionForUnreadableDir() throws Exception Path testRootDir = getTestRootPath(fSys, "test/hadoop/dir"); Path obscuredDir = new Path(testRootDir, "foo"); // so foo is non-empty Path subDir = new Path(obscuredDir, "bar"); fSys.mkdirs(subDir); // no access fSys.setPermission(obscuredDir, new FsPermission((short) 0)); try { fSys.listStatus(obscuredDir); Assert.fail("Should throw IOException"); } catch (IOException ioe) { // expected } finally { // make sure the test directory can be deleted // default fSys.setPermission(obscuredDir, new FsPermission((short) 0755)); }
6076 -1009652557apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 TODO: sample the generated key/value records, and put the numbers below TODO: sample the generated key/value records, and put the numbers below FILE_PATH_CHANGED setUp() public void setUp() throws IOException skip = !(Algorithm.LZO.isSupported()); if (skip) { System.out.println("Skipped"); } // TODO: sample the generated key/value records, and put the numbers below init(Compression.Algorithm.LZO.getName(), "memcmp", "TFileTestCodecsLzo", 2605, 2558); if (!skip) super.setUp();
6075 -1009652587apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 move over the separator for next search move over the separator for next search FILE_PATH_CHANGED split(String, char, char) public static String[] split(String str, char escapeChar, char separator) if (str == null) { return null; } ArrayList strList = new ArrayList(); StringBuilder split = new StringBuilder(); int index = 0; while ((index = findNext(str, separator, escapeChar, index, split)) >= 0) { // move over the separator for next search ++index; strList.add(split.toString()); // reset the buffer split.setLength(0); } strList.add(split.toString()); // remove trailing empty split(s) // last split int last = strList.size(); while (--last >= 0 && "".equals(strList.get(last))) { strList.remove(last); } return strList.toArray(new String[strList.size()]);
6074 -1009652544apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 Some filesystem like HDFS ignore the \"x\" bit if the permission. Others like localFs does not. Override the method below if the file system being tested masks our certain bits for file masks. Some filesystem like HDFS ignore the \"x\" bit if the permission. Others like localFs does not. Override the method below if the file system being tested masks our certain bits for file masks. FILE_PATH_CHANGED setUp() public void setUp() throws Exception fc.mkdir(getTestRootPath(fc), FileContext.DEFAULT_PERM, true);
6073 -1009652582apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 give up give up FILE_PATH_CHANGED sortInternal(IndexedSortable, int, int, Progressable, int) private static void sortInternal(final IndexedSortable s, int p, int r, final Progressable rep, int depth) if (null != rep) { rep.progress(); } while (true) { if (r - p < 13) { for (int i = p; i < r; ++i) { for (int j = i; j > p && s.compare(j - 1, j) > 0; --j) { s.swap(j, j - 1); } } return; } if (--depth < 0) { // give up alt.sort(s, p, r, rep); return; } // select, move pivot into first position fix(s, (p + r) >>> 1, p); fix(s, (p + r) >>> 1, r - 1); fix(s, p, r - 1); // Divide int i = p; int j = r; int ll = p; int rr = r; int cr; while (true) { while (++i < j) { if ((cr = s.compare(i, p)) > 0) break; if (0 == cr && ++ll != i) { s.swap(ll, i); } } while (--j > i) { if ((cr = s.compare(p, j)) > 0) break; if (0 == cr && --rr != j) { s.swap(rr, j); } } if (i < j) s.swap(i, j); else break; } j = i; // swap pivot- and all eq values- into position while (ll >= p) { s.swap(ll--, --i); } while (rr < r) { s.swap(rr++, j++); } // Conquer // Recurse on smaller interval first to keep stack shallow assert i != j; if (i - p < r - j) { sortInternal(s, p, i, rep, depth); p = j; } else { sortInternal(s, j, r, rep, depth); r = i; } }
6072 -1009652479apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 Ignored the mbean itself was not found, which should never happen because we just accessed it (perhaps something unregistered in-between) but if this happens just don't output the attribute. Ignored the mbean itself was not found, which should never happen because we just accessed it (perhaps something unregistered in-between) but if this happens just don't output the attribute. FILE_PATH_CHANGED writeAttribute(JsonGenerator, ObjectName, MBeanAttributeInfo) private void writeAttribute(JsonGenerator jg, ObjectName oname, MBeanAttributeInfo attr) throws IOException if (!attr.isReadable()) { return; } String attName = attr.getName(); if ("modelerType".equals(attName)) { return; } if (attName.indexOf("=") >= 0 || attName.indexOf(":") >= 0 || attName.indexOf(" ") >= 0) { return; } Object value = null; try { value = mBeanServer.getAttribute(oname, attName); } catch (AttributeNotFoundException e) { // Ignored the attribute was not found, which should never happen because the bean // just told us that it has this attribute, but if this happens just don't output // the attribute. return; } catch (MBeanException e) { // The code inside the attribute getter threw an exception so log it, and // skip outputting the attribute LOG.error("getting attribute " + attName + " of " + oname + " threw an exception", e); return; } catch (RuntimeException e) { // For some reason even with an MBeanException available to them Runtime exceptions // can still find their way through, so treat them the same as MBeanException LOG.error("getting attribute " + attName + " of " + oname + " threw an exception", e); return; } catch (ReflectionException e) { // This happens when the code inside the JMX bean (setter?? from the java docs) // threw an exception, so log it and skip outputting the attribute LOG.error("getting attribute " + attName + " of " + oname + " threw an exception", e); return; } catch (InstanceNotFoundException e) { // Ignored the mbean itself was not found, which should never happen because we // just accessed it (perhaps something unregistered in-between) but if this // happens just don't output the attribute. return; } writeAttribute(jg, attName, value);
6071 -1009652121apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 For some reason even with an MBeanException available to them Runtime exceptionscan still find their way through, so treat them the same as MBeanException For some reason even with an MBeanException available to them Runtime exceptionscan still find their way through, so treat them the same as MBeanException FILE_PATH_CHANGED listBeans(JsonGenerator, ObjectName, String, HttpServletResponse) private void listBeans(JsonGenerator jg, ObjectName qry, String attribute, HttpServletResponse response) throws IOException LOG.debug("Listing beans for " + qry); Set names = null; names = mBeanServer.queryNames(qry, null); jg.writeArrayFieldStart("beans"); Iterator it = names.iterator(); while (it.hasNext()) { ObjectName oname = it.next(); MBeanInfo minfo; String code = ""; Object attributeinfo = null; try { minfo = mBeanServer.getMBeanInfo(oname); code = minfo.getClassName(); String prs = ""; try { if ("org.apache.commons.modeler.BaseModelMBean".equals(code)) { prs = "modelerType"; code = (String) mBeanServer.getAttribute(oname, prs); } if (attribute != null) { prs = attribute; attributeinfo = mBeanServer.getAttribute(oname, prs); } } catch (AttributeNotFoundException e) { // If the modelerType attribute was not found, the class name is used // instead. LOG.error("getting attribute " + prs + " of " + oname + " threw an exception", e); } catch (MBeanException e) { // The code inside the attribute getter threw an exception so log it, // and fall back on the class name LOG.error("getting attribute " + prs + " of " + oname + " threw an exception", e); } catch (RuntimeException e) { // For some reason even with an MBeanException available to them // Runtime exceptionscan still find their way through, so treat them // the same as MBeanException LOG.error("getting attribute " + prs + " of " + oname + " threw an exception", e); } catch (ReflectionException e) { // This happens when the code inside the JMX bean (setter?? from the // java docs) threw an exception, so log it and fall back on the // class name LOG.error("getting attribute " + prs + " of " + oname + " threw an exception", e); } } catch (InstanceNotFoundException e) { // Ignored for some reason the bean was not found so don't output it continue; } catch (IntrospectionException e) { // This is an internal error, something odd happened with reflection so // log it and don't output the bean. LOG.error("Problem while trying to process JMX query: " + qry + " with MBean " + oname, e); continue; } catch (ReflectionException e) { // This happens when the code inside the JMX bean threw an exception, so // log it and don't output the bean. LOG.error("Problem while trying to process JMX query: " + qry + " with MBean " + oname, e); continue; } jg.writeStartObject(); jg.writeStringField("name", oname.toString()); jg.writeStringField("modelerType", code); if ((attribute != null) && (attributeinfo == null)) { jg.writeStringField("result", "ERROR"); jg.writeStringField("message", "No attribute with name " + attribute + " was found."); jg.writeEndObject(); jg.writeEndArray(); jg.close(); response.setStatus(HttpServletResponse.SC_NOT_FOUND); return; } if (attribute != null) { writeAttribute(jg, attribute, attributeinfo); } else { MBeanAttributeInfo[] attrs = minfo.getAttributes(); for (int i = 0; i < attrs.length; i++) { writeAttribute(jg, oname, attrs[i]); } } jg.writeEndObject(); } jg.writeEndArray();
6070 -1009652499apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 TO DO: - more efficient to not split the path, but simply compare TO DO: - more efficient to not split the path, but simply compare FILE_PATH_CHANGED resolve(String, boolean) // TO DO: - more efficient to not split the path, but simply compare String[] path = breakIntoPathComponents(p); if (path.length <= 1) { // special case for when path is "/" ResolveResult res = new ResolveResult(ResultKind.isInternalDir, root.InodeDirFs, root.fullPath, SlashPath); return res; } INodeDir curInode = root; int i; // ignore first slash for (i = 1; i < path.length - (resolveLastComponent ? 0 : 1); i++) { INode nextInode = curInode.resolveInternal(path[i]); if (nextInode == null) { StringBuilder failedAt = new StringBuilder(path[0]); for (int j = 1; j <= i; ++j) { failedAt.append('/').append(path[j]); } throw (new FileNotFoundException(failedAt.toString())); } if (nextInode instanceof INodeLink) { final INodeLink link = (INodeLink) nextInode; final Path remainingPath; if (i >= path.length - 1) { remainingPath = SlashPath; } else { StringBuilder remainingPathStr = new StringBuilder("/" + path[i + 1]); for (int j = i + 2; j < path.length; ++j) { remainingPathStr.append('/').append(path[j]); } remainingPath = new Path(remainingPathStr.toString()); } final ResolveResult res = new ResolveResult(ResultKind.isExternalDir, link.targetFileSystem, nextInode.fullPath, remainingPath); return res; } else if (nextInode instanceof INodeDir) { curInode = (INodeDir) nextInode; } } // We have resolved to an internal dir in mount table. Path remainingPath; if (resolveLastComponent) { remainingPath = SlashPath; } else { // note we have taken care of when path is "/" above // for internal dirs rem-path does not start with / since the lookup // that follows will do a children.get(remaningPath) and will have to // strip-out the initial / StringBuilder remainingPathStr = new StringBuilder("/" + path[i]); for (int j = i + 1; j < path.length; ++j) { remainingPathStr.append('/').append(path[j]); } remainingPath = new Path(remainingPathStr.toString()); } final ResolveResult res = new ResolveResult(ResultKind.isInternalDir, curInode.InodeDirFs, curInode.fullPath, remainingPath); return res; ResolveResult resolve(final String p, final boolean resolveLastComponent) throws FileNotFoundException
6069 -1009652496apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 Test resolvePath(p) TODO In the tests below replace fcView.getDefaultFileSystem().resolvePath() fcView.resolvePath() Test resolvePath(p) TODO In the tests below replace fcView.getDefaultFileSystem().resolvePath() fcView.resolvePath() FILE_PATH_CHANGED testResolvePathInternalPaths() public void testResolvePathInternalPaths() throws IOException Assert.assertEquals(new Path("/"), fcView.resolvePath(new Path("/"))); Assert.assertEquals(new Path("/internalDir"), fcView.resolvePath(new Path("/internalDir")));
6068 -1009652529apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 NOTE: this is the one place we do NOT want to save the number of bytes sent (nRemaining here) into lastBytesSent: since we are resending what we've already sent before, offset is nonzero in general (only way it could be zero is if it already equals nRemaining), which would then screw up the offset calculation _next_ time around. IOW, getRemaining() is in terms of the original, zero-offset bufferload, so lastBytesSent must be as well. Cheesy ASCII art: <------------ m, lastBytesSent -----------> +===============================================+ buffer: |1111111111|22222222222222222|333333333333| | +===============================================+ #1: <-- off -->|<-------- nRemaining ---------> #2: <----------- off ----------->|<-- nRem. --> NOTE: this is the one place we do NOT want to save the number of bytes sent (nRemaining here) into lastBytesSent: since we are resending what we've already sent before, offset is nonzero in general (only way it could be zero is if it already equals nRemaining), which would then screw up the offset calculation _next_ time around. IOW, getRemaining() is in terms of the original, zero-offset bufferload, so lastBytesSent must be as well. Cheesy ASCII art: <------------ m, lastBytesSent -----------> +===============================================+ buffer: |1111111111|22222222222222222|333333333333| | +===============================================+ #1: <-- off -->|<-------- nRemaining ---------> #2: <----------- off ----------->|<-- nRem. --> FILE_PATH_CHANGED read() public int read() throws IOException checkStream(); return (read(oneByte, 0, oneByte.length) == -1) ? -1 : (oneByte[0] & 0xff);
6067 -1009652592apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 Workaround end Workaround end FILE_PATH_CHANGED createBaseListener(Configuration) public Connector createBaseListener(Configuration conf) throws IOException return HttpServer.createDefaultChannelConnector();
6066 -1009652593apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 Workaround to handle the problem reported in HADOOP-4744 Workaround to handle the problem reported in HADOOP-4744 FILE_PATH_CHANGED createBaseListener(Configuration) public Connector createBaseListener(Configuration conf) throws IOException return HttpServer.createDefaultChannelConnector();
6065 -1009652546apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 Set new default probability if specified through a system.property If neither is specified set default probability to DEFAULT_PROB Set new default probability if specified through a system.property If neither is specified set default probability to DEFAULT_PROB FILE_PATH_CHANGED injectCriteria(String) public static boolean injectCriteria(String klassName) boolean trigger = false; if (generator.nextFloat() < getProbability(klassName)) { trigger = true; } return trigger;
6063 -1009652585apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 move checksum file too move checksum file too FILE_PATH_CHANGED reportChecksumFailure(Path, FSDataInputStream, long, FSDataInputStream, long) public boolean reportChecksumFailure(Path p, FSDataInputStream in, long inPos, FSDataInputStream sums, long sumsPos) try { // canonicalize f File f = ((RawLocalFileSystem) fs).pathToFile(p).getCanonicalFile(); // find highest writable parent dir of f on the same device String device = new DF(f, getConf()).getMount(); File parent = f.getParentFile(); File dir = null; while (parent != null && parent.canWrite() && parent.toString().startsWith(device)) { dir = parent; parent = parent.getParentFile(); } if (dir == null) { throw new IOException("not able to find the highest writable parent dir"); } // move the file there File badDir = new File(dir, "bad_files"); if (!badDir.mkdirs()) { if (!badDir.isDirectory()) { throw new IOException("Mkdirs failed to create " + badDir.toString()); } } String suffix = "." + rand.nextInt(); File badFile = new File(badDir, f.getName() + suffix); LOG.warn("Moving bad file " + f + " to " + badFile); // close it first in.close(); // rename it boolean b = f.renameTo(badFile); if (!b) { LOG.warn("Ignoring failure of renameTo"); } // move checksum file too File checkFile = ((RawLocalFileSystem) fs).pathToFile(getChecksumFile(p)); b = checkFile.renameTo(new File(badDir, checkFile.getName() + suffix)); if (!b) { LOG.warn("Ignoring failure of renameTo"); } } catch (IOException e) { LOG.warn("Error moving bad file " + p + ": " + e); } return false;
6062 -1009652495apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 hack to avoid most of the \"innocuous\" races. hack to avoid most of the \"innocuous\" races. FILE_PATH_CHANGED start() void start() if (startMBeans) startMBeans();
6061 -1009652522apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 better safe than sorry (should never happen) better safe than sorry (should never happen) FILE_PATH_CHANGED cacheGroupsAdd(List) public void cacheGroupsAdd(List groups) throws IOException for (String group : groups) { if (group.length() == 0) { // better safe than sorry (should never happen) } else if (group.charAt(0) == '@') { if (!NetgroupCache.isCached(group)) { NetgroupCache.add(group, getUsersForNetgroup(group)); } } else { // unix group, not caching } }
6060 -1009652578apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 Send it to S3 TODO: Use passed in Progressable to report progress. Send it to S3 TODO: Use passed in Progressable to report progress. FILE_PATH_CHANGED endBlock() private synchronized void endBlock() throws IOException // // Done with local copy // backupStream.close(); // // Send it to S3 // // TODO: Use passed in Progressable to report progress. nextBlockOutputStream(); store.storeBlock(nextBlock, backupFile); internalClose(); // // Delete local backup, start new one // boolean b = backupFile.delete(); if (!b) { LOG.warn("Ignoring failed delete"); } backupFile = newBackupFile(); backupStream = new FileOutputStream(backupFile); bytesWrittenToBlock = 0;
6059 -1009652527apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 Even though tmpDir is dangling symlink to tmp, fullyDelete(tmpDir) should delete tmpDir properly. Even though tmpDir is dangling symlink to tmp, fullyDelete(tmpDir) should delete tmpDir properly. FILE_PATH_CHANGED setupDirs() private void setupDirs() throws IOException Assert.assertFalse(del.exists()); Assert.assertFalse(tmp.exists()); Assert.assertFalse(partitioned.exists()); del.mkdirs(); tmp.mkdirs(); partitioned.mkdirs(); new File(del, FILE).createNewFile(); File tmpFile = new File(tmp, FILE); tmpFile.createNewFile(); // create directories dir1.mkdirs(); dir2.mkdirs(); new File(dir1, FILE).createNewFile(); new File(dir2, FILE).createNewFile(); // create a symlink to file File link = new File(del, LINK); FileUtil.symLink(tmpFile.toString(), link.toString()); // create a symlink to dir File linkDir = new File(del, "tmpDir"); FileUtil.symLink(tmp.toString(), linkDir.toString()); Assert.assertEquals(5, del.listFiles().length); // create files in partitioned directories createFile(partitioned, "part-r-00000", "foo"); createFile(partitioned, "part-r-00001", "bar");
6058 -1009652528apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 Even though 'y' is dangling symlink to file tmp/x, fullyDelete(y) should delete 'y' properly. Even though 'y' is dangling symlink to file tmp/x, fullyDelete(y) should delete 'y' properly. FILE_PATH_CHANGED setupDirs() private void setupDirs() throws IOException Assert.assertFalse(del.exists()); Assert.assertFalse(tmp.exists()); Assert.assertFalse(partitioned.exists()); del.mkdirs(); tmp.mkdirs(); partitioned.mkdirs(); new File(del, FILE).createNewFile(); File tmpFile = new File(tmp, FILE); tmpFile.createNewFile(); // create directories dir1.mkdirs(); dir2.mkdirs(); new File(dir1, FILE).createNewFile(); new File(dir2, FILE).createNewFile(); // create a symlink to file File link = new File(del, LINK); FileUtil.symLink(tmpFile.toString(), link.toString()); // create a symlink to dir File linkDir = new File(del, "tmpDir"); FileUtil.symLink(tmp.toString(), linkDir.toString()); Assert.assertEquals(5, del.listFiles().length); // create files in partitioned directories createFile(partitioned, "part-r-00000", "foo"); createFile(partitioned, "part-r-00001", "bar");
6057 -1009652481apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 NOTE: this logic should be better, mimics previous implementation NOTE: this logic should be better, mimics previous implementation FILE_PATH_CHANGED registerCommands(CommandFactory) public static void registerCommands(CommandFactory factory) factory.addClass(Merge.class, "-getmerge"); factory.addClass(Cp.class, "-cp"); factory.addClass(CopyFromLocal.class, "-copyFromLocal"); factory.addClass(CopyToLocal.class, "-copyToLocal"); factory.addClass(Get.class, "-get"); factory.addClass(Put.class, "-put");
6056 -1009652482apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 too bad we can't tell exactly why it failed... too bad we can't tell exactly why it failed... FILE_PATH_CHANGED registerCommands(CommandFactory) public static void registerCommands(CommandFactory factory) factory.addClass(Merge.class, "-getmerge"); factory.addClass(Cp.class, "-cp"); factory.addClass(CopyFromLocal.class, "-copyFromLocal"); factory.addClass(CopyToLocal.class, "-copyToLocal"); factory.addClass(Get.class, "-get"); factory.addClass(Put.class, "-put");
6055 -1009652483apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 TODO: this really should be a -nl option TODO: this really should be a -nl option FILE_PATH_CHANGED registerCommands(CommandFactory) public static void registerCommands(CommandFactory factory) factory.addClass(Merge.class, "-getmerge"); factory.addClass(Cp.class, "-cp"); factory.addClass(CopyFromLocal.class, "-copyFromLocal"); factory.addClass(CopyToLocal.class, "-copyToLocal"); factory.addClass(Get.class, "-get"); factory.addClass(Put.class, "-put");
6054 -1009652525apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 Check if zlib consumed all input buffer set keepUncompressedBuf properly zlib consumed all input buffer Check if zlib consumed all input buffer set keepUncompressedBuf properly zlib consumed all input buffer FILE_PATH_CHANGED compress(byte[], int, int) public synchronized int compress(byte[] b, int off, int len) throws IOException if (b == null) { throw new NullPointerException(); } if (off < 0 || len < 0 || off > b.length - len) { throw new ArrayIndexOutOfBoundsException(); } int n = 0; // Check if there is compressed data n = compressedDirectBuf.remaining(); if (n > 0) { n = Math.min(n, len); ((ByteBuffer) compressedDirectBuf).get(b, off, n); return n; } // Re-initialize the zlib's output direct buffer compressedDirectBuf.rewind(); compressedDirectBuf.limit(directBufferSize); // Compress data n = deflateBytesDirect(); compressedDirectBuf.limit(n); // Check if zlib consumed all input buffer // set keepUncompressedBuf properly if (uncompressedDirectBufLen <= 0) { // zlib consumed all input buffer keepUncompressedBuf = false; uncompressedDirectBuf.clear(); uncompressedDirectBufOff = 0; uncompressedDirectBufLen = 0; } else { // zlib did not consume all input buffer keepUncompressedBuf = true; } // Get atmost 'len' bytes n = Math.min(n, len); ((ByteBuffer) compressedDirectBuf).get(b, off, n); return n;
6052 -1009652577apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 walk through the list, searching. Not the most efficient way, but this in intended to be used rarely, so we keep it simple. As an optimization, we can keep a hashmap of record name to its RTI, for later. walk through the list, searching. Not the most efficient way, but this in intended to be used rarely, so we keep it simple. As an optimization, we can keep a hashmap of record name to its RTI, for later. FILE_PATH_CHANGED findStruct(String) // walk through the list, searching. Not the most efficient way, but this // in intended to be used rarely, so we keep it simple. // As an optimization, we can keep a hashmap of record name to its RTI, for later. for (FieldTypeInfo ti : typeInfos) { if ((0 == ti.getFieldID().compareTo(name)) && (ti.getTypeID().getTypeVal() == RIOType.STRUCT)) { return (StructTypeID) ti.getTypeID(); } } return null; StructTypeID findStruct(String name)
6050 -1009652536apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 Authorization is done. Just call super. Authorization is done. Just call super. FILE_PATH_CHANGED doGet(HttpServletRequest, HttpServletResponse) protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException // Do the authorization if (HttpServer.hasAdministratorAccess(getServletContext(), request, response)) { // Authorization is done. Just call super. super.doGet(request, response); }
6049 -1009652579apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 Iterate up to N_ITERS times to improve the tables. Iterate up to N_ITERS times to improve the tables. FILE_PATH_CHANGED hbMakeCodeLengths(char[], int[], int, int) protected static void hbMakeCodeLengths(char[] len, int[] freq, int alphaSize, int maxLen) /* * Nodes and heap entries run from 1. Entry 0 for both the heap and * nodes is a sentinel. */ final int[] heap = new int[MAX_ALPHA_SIZE * 2]; final int[] weight = new int[MAX_ALPHA_SIZE * 2]; final int[] parent = new int[MAX_ALPHA_SIZE * 2]; for (int i = alphaSize; --i >= 0; ) { weight[i + 1] = (freq[i] == 0 ? 1 : freq[i]) << 8; } for (boolean tooLong = true; tooLong; ) { tooLong = false; int nNodes = alphaSize; int nHeap = 0; heap[0] = 0; weight[0] = 0; parent[0] = -2; for (int i = 1; i <= alphaSize; i++) { parent[i] = -1; nHeap++; heap[nHeap] = i; int zz = nHeap; int tmp = heap[zz]; while (weight[tmp] < weight[heap[zz >> 1]]) { heap[zz] = heap[zz >> 1]; zz >>= 1; } heap[zz] = tmp; } // assert (nHeap < (MAX_ALPHA_SIZE + 2)) : nHeap; while (nHeap > 1) { int n1 = heap[1]; heap[1] = heap[nHeap]; nHeap--; int yy = 0; int zz = 1; int tmp = heap[1]; while (true) { yy = zz << 1; if (yy > nHeap) { break; } if ((yy < nHeap) && (weight[heap[yy + 1]] < weight[heap[yy]])) { yy++; } if (weight[tmp] < weight[heap[yy]]) { break; } heap[zz] = heap[yy]; zz = yy; } heap[zz] = tmp; int n2 = heap[1]; heap[1] = heap[nHeap]; nHeap--; yy = 0; zz = 1; tmp = heap[1]; while (true) { yy = zz << 1; if (yy > nHeap) { break; } if ((yy < nHeap) && (weight[heap[yy + 1]] < weight[heap[yy]])) { yy++; } if (weight[tmp] < weight[heap[yy]]) { break; } heap[zz] = heap[yy]; zz = yy; } heap[zz] = tmp; nNodes++; parent[n1] = parent[n2] = nNodes; final int weight_n1 = weight[n1]; final int weight_n2 = weight[n2]; weight[nNodes] = (((weight_n1 & 0xffffff00) + (weight_n2 & 0xffffff00)) | (1 + (((weight_n1 & 0x000000ff) > (weight_n2 & 0x000000ff)) ? (weight_n1 & 0x000000ff) : (weight_n2 & 0x000000ff)))); parent[nNodes] = -1; nHeap++; heap[nHeap] = nNodes; tmp = 0; zz = nHeap; tmp = heap[zz]; final int weight_tmp = weight[tmp]; while (weight_tmp < weight[heap[zz >> 1]]) { heap[zz] = heap[zz >> 1]; zz >>= 1; } heap[zz] = tmp; } // assert (nNodes < (MAX_ALPHA_SIZE * 2)) : nNodes; for (int i = 1; i <= alphaSize; i++) { int j = 0; int k = i; for (int parent_k; (parent_k = parent[k]) >= 0; ) { k = parent_k; j++; } len[i - 1] = (char) j; if (j > maxLen) { tooLong = true; } } if (tooLong) { for (int i = 1; i < alphaSize; i++) { int j = weight[i] >> 8; j = 1 + (j >> 1); weight[i] = j << 8; } } }
6048 -1009652580apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 Now another magic 48-bit number, 0x177245385090, to indicate the end of the last block. (sqrt(pi), if you want to know. I did want to use e, but it contains too much repetition -- 27 18 28 18 28 46 -- for me to feel statistically comfortable. Call me paranoid.) Now another magic 48-bit number, 0x177245385090, to indicate the end of the last block. (sqrt(pi), if you want to know. I did want to use e, but it contains too much repetition -- 27 18 28 18 28 46 -- for me to feel statistically comfortable. Call me paranoid.) FILE_PATH_CHANGED hbMakeCodeLengths(char[], int[], int, int) protected static void hbMakeCodeLengths(char[] len, int[] freq, int alphaSize, int maxLen) /* * Nodes and heap entries run from 1. Entry 0 for both the heap and * nodes is a sentinel. */ final int[] heap = new int[MAX_ALPHA_SIZE * 2]; final int[] weight = new int[MAX_ALPHA_SIZE * 2]; final int[] parent = new int[MAX_ALPHA_SIZE * 2]; for (int i = alphaSize; --i >= 0; ) { weight[i + 1] = (freq[i] == 0 ? 1 : freq[i]) << 8; } for (boolean tooLong = true; tooLong; ) { tooLong = false; int nNodes = alphaSize; int nHeap = 0; heap[0] = 0; weight[0] = 0; parent[0] = -2; for (int i = 1; i <= alphaSize; i++) { parent[i] = -1; nHeap++; heap[nHeap] = i; int zz = nHeap; int tmp = heap[zz]; while (weight[tmp] < weight[heap[zz >> 1]]) { heap[zz] = heap[zz >> 1]; zz >>= 1; } heap[zz] = tmp; } // assert (nHeap < (MAX_ALPHA_SIZE + 2)) : nHeap; while (nHeap > 1) { int n1 = heap[1]; heap[1] = heap[nHeap]; nHeap--; int yy = 0; int zz = 1; int tmp = heap[1]; while (true) { yy = zz << 1; if (yy > nHeap) { break; } if ((yy < nHeap) && (weight[heap[yy + 1]] < weight[heap[yy]])) { yy++; } if (weight[tmp] < weight[heap[yy]]) { break; } heap[zz] = heap[yy]; zz = yy; } heap[zz] = tmp; int n2 = heap[1]; heap[1] = heap[nHeap]; nHeap--; yy = 0; zz = 1; tmp = heap[1]; while (true) { yy = zz << 1; if (yy > nHeap) { break; } if ((yy < nHeap) && (weight[heap[yy + 1]] < weight[heap[yy]])) { yy++; } if (weight[tmp] < weight[heap[yy]]) { break; } heap[zz] = heap[yy]; zz = yy; } heap[zz] = tmp; nNodes++; parent[n1] = parent[n2] = nNodes; final int weight_n1 = weight[n1]; final int weight_n2 = weight[n2]; weight[nNodes] = (((weight_n1 & 0xffffff00) + (weight_n2 & 0xffffff00)) | (1 + (((weight_n1 & 0x000000ff) > (weight_n2 & 0x000000ff)) ? (weight_n1 & 0x000000ff) : (weight_n2 & 0x000000ff)))); parent[nNodes] = -1; nHeap++; heap[nHeap] = nNodes; tmp = 0; zz = nHeap; tmp = heap[zz]; final int weight_tmp = weight[tmp]; while (weight_tmp < weight[heap[zz >> 1]]) { heap[zz] = heap[zz >> 1]; zz >>= 1; } heap[zz] = tmp; } // assert (nNodes < (MAX_ALPHA_SIZE * 2)) : nNodes; for (int i = 1; i <= alphaSize; i++) { int j = 0; int k = i; for (int parent_k; (parent_k = parent[k]) >= 0; ) { k = parent_k; j++; } len[i - 1] = (char) j; if (j > maxLen) { tooLong = true; } } if (tooLong) { for (int i = 1; i < alphaSize; i++) { int j = weight[i] >> 8; j = 1 + (j >> 1); weight[i] = j << 8; } } }
6046 -1009652599apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 maybe too long maybe too long FILE_PATH_CHANGED writeString(DataOutput, String) public static int writeString(DataOutput out, String s) throws IOException if (s.length() > 0xffff / 3) { // maybe too long LOG.warn("truncating long string: " + s.length() + " chars, starting with " + s.substring(0, 20)); s = s.substring(0, 0xffff / 3); } int len = utf8Length(s); if (// double-check length len > 0xffff) throw new IOException("string too long!"); out.writeShort(len); writeChars(out, s, 0, s.length()); return len;
6045 -1009652600apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 maybe too long maybe too long FILE_PATH_CHANGED set(String) public void set(String string) if (string.length() > 0xffff / 3) { // maybe too long LOG.warn("truncating long string: " + string.length() + " chars, starting with " + string.substring(0, 20)); string = string.substring(0, 0xffff / 3); } // compute length length = utf8Length(string); if (// double-check length length > 0xffff) throw new RuntimeException("string too long!"); if (// grow buffer bytes == null || length > bytes.length) bytes = new byte[length]; try { // avoid sync'd allocations DataOutputBuffer obuf = OBUF_FACTORY.get(); obuf.reset(); writeChars(obuf, string, 0, string.length()); System.arraycopy(obuf.getData(), 0, bytes, 0, length); } catch (IOException e) { throw new RuntimeException(e); }
6044 -1009652606apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 If they try to bind to a different host's address, give a better error message. If they try to bind to a different host's address, give a better error message. FILE_PATH_CHANGED getProtocolClass(String, Configuration) static Class getProtocolClass(String protocolName, Configuration conf) throws ClassNotFoundException Class protocol = PROTOCOL_CACHE.get(protocolName); if (protocol == null) { protocol = conf.getClassByName(protocolName); PROTOCOL_CACHE.put(protocolName, protocol); } return protocol;
6042 -1009652560apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 the directory better be empty the directory better be empty FILE_PATH_CHANGED rmdir(String) public int rmdir(String path) throws IOException if (isDirectory(path)) { // the directory better be empty String[] dirEntries = readdir(path); if ((dirEntries.length <= 2) && (localFS.delete(new Path(path), true))) return 0; } return -1;
6041 -1009652506apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 implicitly recurse once for cmdline directories implicitly recurse once for cmdline directories FILE_PATH_CHANGED registerCommands(CommandFactory) public static void registerCommands(CommandFactory factory) factory.addClass(Ls.class, "-ls"); factory.addClass(Lsr.class, "-lsr");
6040 -1009652501apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 this is very ugly, but needed to avoid breaking hdfs tests... if a path has no authority, then the FileStatus from globStatus will add the \"-fs\" authority into the path, so we need to sub it back out to satisfy the tests this is very ugly, but needed to avoid breaking hdfs tests... if a path has no authority, then the FileStatus from globStatus will add the \"-fs\" authority into the path, so we need to sub it back out to satisfy the tests FILE_PATH_CHANGED expandAsGlob(String, Configuration) public static PathData[] expandAsGlob(String pattern, Configuration conf) throws IOException Path globPath = new Path(pattern); FileSystem fs = globPath.getFileSystem(conf); FileStatus[] stats = fs.globStatus(globPath); PathData[] items = null; if (stats == null) { // not a glob & file not found, so add the path with a null stat items = new PathData[] { new PathData(fs, pattern, null) }; } else if (// this is very ugly, but needed to avoid breaking hdfs tests... // if a path has no authority, then the FileStatus from globStatus // will add the "-fs" authority into the path, so we need to sub // it back out to satisfy the tests stats.length == 1 && stats[0].getPath().equals(fs.makeQualified(globPath))) { // if the fq path is identical to the pattern passed, use the pattern // to initialize the string value items = new PathData[] { new PathData(fs, pattern, stats[0]) }; } else { // convert stats to PathData items = new PathData[stats.length]; int i = 0; for (FileStatus stat : stats) { items[i++] = new PathData(fs, stat); } } return items;
6039 -1009652476apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 TODO: this should be abstract in a base class TODO: this should be abstract in a base class FILE_PATH_CHANGED getFS() protected FileSystem getFS() throws IOException if (fs == null) { fs = FileSystem.get(getConf()); } return fs;
6038 -1009652477apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 instance.run catches IOE, so something is REALLY wrong if here instance.run catches IOE, so something is REALLY wrong if here FILE_PATH_CHANGED getFS() protected FileSystem getFS() throws IOException if (fs == null) { fs = FileSystem.get(getConf()); } return fs;
6037 -1009652502apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 TODO: will eventually auto-wrap the text, but this matches the expected output for the hdfs tests... TODO: will eventually auto-wrap the text, but this matches the expected output for the hdfs tests... FILE_PATH_CHANGED getFS() protected FileSystem getFS() throws IOException if (fs == null) { fs = FileSystem.get(getConf()); } return fs;
6036 -1009652478apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 TODO: DFSAdmin subclasses FsShell so need to protect the command registration. This class should morph into a base class for commands, and then this method can be abstract TODO: DFSAdmin subclasses FsShell so need to protect the command registration. This class should morph into a base class for commands, and then this method can be abstract FILE_PATH_CHANGED getFS() protected FileSystem getFS() throws IOException if (fs == null) { fs = FileSystem.get(getConf()); } return fs;
6035 -1009652595apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 Ideally we should wait after transferTo returns 0. But because of a bug in JRE on Linux (http://bugs.sun.com/view_bug.do?bug_id=5103988), which throws an exception instead of returning 0, we wait for the channel to be writable before writing to it. If you ever see IOException with message \"Resource temporarily unavailable\" thrown here, please let us know. Once we move to JAVA SE 7, wait should be moved to correct place. Ideally we should wait after transferTo returns 0. But because of a bug in JRE on Linux (http://bugs.sun.com/view_bug.do?bug_id=5103988), which throws an exception instead of returning 0, we wait for the channel to be writable before writing to it. If you ever see IOException with message \"Resource temporarily unavailable\" thrown here, please let us know. Once we move to JAVA SE 7, wait should be moved to correct place. FILE_PATH_CHANGED transferToFully(FileChannel, long, int) public void transferToFully(FileChannel fileCh, long position, int count) throws IOException while (count > 0) { /* * Ideally we should wait after transferTo returns 0. But because of * a bug in JRE on Linux (http://bugs.sun.com/view_bug.do?bug_id=5103988), * which throws an exception instead of returning 0, we wait for the * channel to be writable before writing to it. If you ever see * IOException with message "Resource temporarily unavailable" * thrown here, please let us know. * * Once we move to JAVA SE 7, wait should be moved to correct place. */ waitForWritable(); int nTransfered = (int) fileCh.transferTo(position, count, getChannel()); if (nTransfered == 0) { // check if end of file is reached. if (position >= fileCh.size()) { throw new EOFException("EOF Reached. file size is " + fileCh.size() + " and " + count + " more bytes left to be " + "transfered."); } // otherwise assume the socket is full. // waitForWritable(); // see comment above. } else if (nTransfered < 0) { throw new IOException("Unexpected return of " + nTransfered + " from transferTo()"); } else { position += nTransfered; count -= nTransfered; } }
6034 -1009652545apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 Default impl assumes that permissions do not matter calling the regular create is good enough. FSs that implement permissions should override this. Default impl assumes that permissions do not matter calling the regular create is good enough. FSs that implement permissions should override this. FILE_PATH_CHANGED createInternal(Path, EnumSet, FsPermission, int, short, long, Progressable, int, boolean) public FSDataOutputStream createInternal(Path f, EnumSet flag, FsPermission absolutePermission, int bufferSize, short replication, long blockSize, Progressable progress, int bytesPerChecksum, boolean createParent) throws IOException checkPath(f); // Default impl assumes that permissions do not matter // calling the regular create is good enough. // FSs that implement permissions should override this. if (!createParent) { // parent must exist. // since this.create makes parent dirs automatically // we must throw exception if parent does not exist. final FileStatus stat = getFileStatus(f.getParent()); if (stat == null) { throw new FileNotFoundException("Missing parent:" + f); } if (!stat.isDirectory()) { throw new ParentNotDirectoryException("parent is not a dir:" + f); } // parent does exist - go ahead with create of file. } return fsImpl.primitiveCreate(f, absolutePermission, flag, bufferSize, replication, blockSize, progress, bytesPerChecksum);
6033 -1009652589apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 Using default block size since there is no way in FTP client to know of block sizes on server. The assumption could be less than ideal. Using default block size since there is no way in FTP client to know of block sizes on server. The assumption could be less than ideal. FILE_PATH_CHANGED getFileStatus(FTPFile, Path) private FileStatus getFileStatus(FTPFile ftpFile, Path parentPath) long length = ftpFile.getSize(); boolean isDir = ftpFile.isDirectory(); int blockReplication = 1; // Using default block size since there is no way in FTP client to know of // block sizes on server. The assumption could be less than ideal. long blockSize = DEFAULT_BLOCK_SIZE; long modTime = ftpFile.getTimestamp().getTimeInMillis(); long accessTime = 0; FsPermission permission = getPermissions(ftpFile); String user = ftpFile.getUser(); String group = ftpFile.getGroup(); Path filePath = new Path(parentPath, ftpFile.getName()); return new FileStatus(length, isDir, blockReplication, blockSize, modTime, accessTime, permission, user, group, filePath.makeQualified(this));
6032 -1009652535apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 perhaps a VIP is failing over perhaps a VIP is failing over FILE_PATH_CHANGED waitForProtocolProxy(Class, long, InetSocketAddress, Configuration, int, long) public static ProtocolProxy waitForProtocolProxy(Class protocol, long clientVersion, InetSocketAddress addr, Configuration conf, int rpcTimeout, long timeout) throws IOException long startTime = System.currentTimeMillis(); IOException ioe; while (true) { try { return getProtocolProxy(protocol, clientVersion, addr, UserGroupInformation.getCurrentUser(), conf, NetUtils.getDefaultSocketFactory(conf), rpcTimeout); } catch (ConnectException se) { // namenode has not been started LOG.info("Server at " + addr + " not available yet, Zzzzz..."); ioe = se; } catch (SocketTimeoutException te) { // namenode is busy LOG.info("Problem connecting to server: " + addr); ioe = te; } catch (NoRouteToHostException nrthe) { // perhaps a VIP is failing over LOG.info("No route to host for server: " + addr); ioe = nrthe; } // check if timed out if (System.currentTimeMillis() - timeout >= startTime) { throw ioe; } // wait for retry try { Thread.sleep(1000); } catch (InterruptedException ie) { // IGNORE } }
6031 -1009652500apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 The implementors of RawLocalFileSystem were trying to be very smart. They implement FileStatus#getOwener lazily -- the object returned is really a RawLocalFileSystem that expect the FileStatus#getPath to be unchanged so that it can get owner when needed. Hence we need to interpose a new ViewFsFileStatus that works around. The implementors of RawLocalFileSystem were trying to be very smart. They implement FileStatus#getOwener lazily -- the object returned is really a RawLocalFileSystem that expect the FileStatus#getPath to be unchanged so that it can get owner when needed. Hence we need to interpose a new ViewFsFileStatus that works around. FILE_PATH_CHANGED readOnlyMountTable(String, String) static AccessControlException readOnlyMountTable(final String operation, final String p) return new AccessControlException("InternalDir of ViewFileSystem is readonly; operation=" + operation + "Path=" + p);
6030 -1009652559apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 Verify fix of bug identified in HADOOP-6004 Verify fix of bug identified in HADOOP-6004 FILE_PATH_CHANGED testDeserialization() public void testDeserialization() throws IOException // Create a test BlockLocation String[] names = { "one", "two" }; String[] hosts = { "three", "four" }; String[] topologyPaths = { "five", "six" }; long offset = 25l; long length = 55l; BlockLocation bl = new BlockLocation(names, hosts, topologyPaths, offset, length); DataOutputBuffer dob = new DataOutputBuffer(); // Serialize it try { bl.write(dob); } catch (IOException e) { fail("Unable to serialize data: " + e.getMessage()); } byte[] bytes = dob.getData(); DataInput da = new DataInputStream(new ByteArrayInputStream(bytes)); // Try to re-create the BlockLocation the same way as is done during // deserialization BlockLocation bl2 = new BlockLocation(); try { bl2.readFields(da); } catch (IOException e) { fail("Unable to deserialize BlockLocation: " + e.getMessage()); } // Check that we got back what we started with verifyDeserialization(bl2.getHosts(), hosts); verifyDeserialization(bl2.getNames(), names); verifyDeserialization(bl2.getTopologyPaths(), topologyPaths); assertEquals(bl2.getOffset(), offset); assertEquals(bl2.getLength(), length);
6029 -1009652533apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 XXX Twenty internal version does not support --script option. XXX Twenty internal version does not support --script option. FILE_PATH_CHANGED getCommand(String, String) protected String[] getCommand(String command, String confDir) ArrayList cmdArgs = new ArrayList(); File binDir = getBinDir(); cmdArgs.add("ssh"); cmdArgs.add(hostName); cmdArgs.add(binDir.getAbsolutePath() + File.separator + SCRIPT_NAME); cmdArgs.add("--config"); cmdArgs.add(confDir); // XXX Twenty internal version does not support --script option. cmdArgs.add(command); cmdArgs.add(daemonName); return (String[]) cmdArgs.toArray(new String[cmdArgs.size()]);
6028 -1009652562apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 Check that every permission has its sticky bit represented correctly Check that every permission has its sticky bit represented correctly FILE_PATH_CHANGED testStickyBitToString() public void testStickyBitToString() // Check that every permission has its sticky bit represented correctly for (boolean sb : new boolean[] { false, true }) { for (FsAction u : FsAction.values()) { for (FsAction g : FsAction.values()) { for (FsAction o : FsAction.values()) { FsPermission f = new FsPermission(u, g, o, sb); if (f.getStickyBit() && f.getOtherAction().implies(EXECUTE)) assertEquals('t', f.toString().charAt(8)); else if (f.getStickyBit() && !f.getOtherAction().implies(EXECUTE)) assertEquals('T', f.toString().charAt(8)); else if (!f.getStickyBit() && f.getOtherAction().implies(EXECUTE)) assertEquals('x', f.toString().charAt(8)); else assertEquals('-', f.toString().charAt(8)); } } } }
6027 -1009652558apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 move the cursor to the beginning of the tail, containing: offset to the meta block index, version and magic move the cursor to the beginning of the tail, containing: offset to the meta block index, version and magic FILE_PATH_CHANGED register(long, long, long) public void register(long raw, long offsetStart, long offsetEnd) mp
6025 -1009652555apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 TODO: remember the longest key in a TFile, and use it to replace MAX_KEY_SIZE. TODO: remember the longest key in a TFile, and use it to replace MAX_KEY_SIZE. FILE_PATH_CHANGED getChunkBufferSize(Configuration) static int getChunkBufferSize(Configuration conf) int ret = conf.getInt(CHUNK_BUF_SIZE_ATTR, 1024 * 1024); return (ret > 0) ? ret : 1024 * 1024;
6024 -1009652556apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 flag to ensure value is only examined once. flag to ensure value is only examined once. FILE_PATH_CHANGED getChunkBufferSize(Configuration) static int getChunkBufferSize(Configuration conf) int ret = conf.getInt(CHUNK_BUF_SIZE_ATTR, 1024 * 1024); return (ret > 0) ? ret : 1024 * 1024;
6023 -1009652487apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 TODO: handle this TODO: handle this FILE_PATH_CHANGED processPath(PathData) protected void processPath(PathData item) throws IOException if (item.stat.isDirectory()) { // TODO: handle this throw new PathIsDirectoryException(item.toString()); } if (item.stat.getLen() != 0) { throw new PathIOException(item.toString(), "Not a zero-length file"); } touchz(item);
6022 -1009652517apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 Read the int[] object as written by ObjectWritable, but \"going around\" ObjectWritable Read the int[] object as written by ObjectWritable, but \"going around\" ObjectWritable FILE_PATH_CHANGED testOldFormat() public void testOldFormat() throws IOException // Make sure we still correctly write the old format if desired. // Write the data array with old ObjectWritable API // which will set allowCompactArrays false. ObjectWritable.writeObject(out, i, i.getClass(), null); // Get ready to read it back in.reset(out.getData(), out.getLength()); // Read the int[] object as written by ObjectWritable, but // "going around" ObjectWritable @SuppressWarnings("deprecation") String className = UTF8.readString(in); assertEquals("The int[] written by ObjectWritable as a non-compact array " + "was not labelled as an array of int", i.getClass().getName(), className); int length = in.readInt(); assertEquals("The int[] written by ObjectWritable as a non-compact array " + "was not expected length", i.length, length); int[] readValue = new int[length]; try { for (int i = 0; i < length; i++) { readValue[i] = (int) ((Integer) ObjectWritable.readObject(in, null)); } } catch (Exception e) { fail("The int[] written by ObjectWritable as a non-compact array " + "was corrupted. Failed to correctly read int[] of length " + length + ". Got exception:\n" + StringUtils.stringifyException(e)); } assertTrue("The int[] written by ObjectWritable as a non-compact array " + "was corrupted.", Arrays.equals(i, readValue));
6021 -1009652518apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 Read the APW object as written by ObjectWritable, but \"going around\" ObjectWritable Read the APW object as written by ObjectWritable, but \"going around\" ObjectWritable FILE_PATH_CHANGED testObjectLabeling() public void testObjectLabeling() throws IOException // Do a few tricky experiments to make sure things are being written // the way we expect // Write the data array with ObjectWritable // which will indirectly write it using APW.Internal ObjectWritable.writeObject(out, i, i.getClass(), null, true); // Write the corresponding APW directly with ObjectWritable ArrayPrimitiveWritable apw = new ArrayPrimitiveWritable(i); ObjectWritable.writeObject(out, apw, apw.getClass(), null, true); // Get ready to read it back in.reset(out.getData(), out.getLength()); // Read the int[] object as written by ObjectWritable, but // "going around" ObjectWritable String className = UTF8.readString(in); assertEquals("The int[] written by ObjectWritable was not labelled as " + "an ArrayPrimitiveWritable.Internal", ArrayPrimitiveWritable.Internal.class.getName(), className); ArrayPrimitiveWritable.Internal apwi = new ArrayPrimitiveWritable.Internal(); apwi.readFields(in); assertEquals("The ArrayPrimitiveWritable.Internal component type was corrupted", int.class, apw.getComponentType()); assertTrue("The int[] written by ObjectWritable as " + "ArrayPrimitiveWritable.Internal was corrupted", Arrays.equals(i, (int[]) (apwi.get()))); // Read the APW object as written by ObjectWritable, but // "going around" ObjectWritable String declaredClassName = UTF8.readString(in); assertEquals("The APW written by ObjectWritable was not labelled as " + "declaredClass ArrayPrimitiveWritable", ArrayPrimitiveWritable.class.getName(), declaredClassName); className = UTF8.readString(in); assertEquals("The APW written by ObjectWritable was not labelled as " + "class ArrayPrimitiveWritable", ArrayPrimitiveWritable.class.getName(), className); ArrayPrimitiveWritable apw2 = new ArrayPrimitiveWritable(); apw2.readFields(in); assertEquals("The ArrayPrimitiveWritable component type was corrupted", int.class, apw2.getComponentType()); assertTrue("The int[] written by ObjectWritable as " + "ArrayPrimitiveWritable was corrupted", Arrays.equals(i, (int[]) (apw2.get())));
6020 -1009652519apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 Read the int[] object as written by ObjectWritable, but \"going around\" ObjectWritable Read the int[] object as written by ObjectWritable, but \"going around\" ObjectWritable FILE_PATH_CHANGED testObjectLabeling() public void testObjectLabeling() throws IOException // Do a few tricky experiments to make sure things are being written // the way we expect // Write the data array with ObjectWritable // which will indirectly write it using APW.Internal ObjectWritable.writeObject(out, i, i.getClass(), null, true); // Write the corresponding APW directly with ObjectWritable ArrayPrimitiveWritable apw = new ArrayPrimitiveWritable(i); ObjectWritable.writeObject(out, apw, apw.getClass(), null, true); // Get ready to read it back in.reset(out.getData(), out.getLength()); // Read the int[] object as written by ObjectWritable, but // "going around" ObjectWritable String className = UTF8.readString(in); assertEquals("The int[] written by ObjectWritable was not labelled as " + "an ArrayPrimitiveWritable.Internal", ArrayPrimitiveWritable.Internal.class.getName(), className); ArrayPrimitiveWritable.Internal apwi = new ArrayPrimitiveWritable.Internal(); apwi.readFields(in); assertEquals("The ArrayPrimitiveWritable.Internal component type was corrupted", int.class, apw.getComponentType()); assertTrue("The int[] written by ObjectWritable as " + "ArrayPrimitiveWritable.Internal was corrupted", Arrays.equals(i, (int[]) (apwi.get()))); // Read the APW object as written by ObjectWritable, but // "going around" ObjectWritable String declaredClassName = UTF8.readString(in); assertEquals("The APW written by ObjectWritable was not labelled as " + "declaredClass ArrayPrimitiveWritable", ArrayPrimitiveWritable.class.getName(), declaredClassName); className = UTF8.readString(in); assertEquals("The APW written by ObjectWritable was not labelled as " + "class ArrayPrimitiveWritable", ArrayPrimitiveWritable.class.getName(), className); ArrayPrimitiveWritable apw2 = new ArrayPrimitiveWritable(); apw2.readFields(in); assertEquals("The ArrayPrimitiveWritable component type was corrupted", int.class, apw2.getComponentType()); assertTrue("The int[] written by ObjectWritable as " + "ArrayPrimitiveWritable was corrupted", Arrays.equals(i, (int[]) (apw2.get())));
6019 -1009652576apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 Following is brittle. Is there a better way? Following is brittle. Is there a better way? FILE_PATH_CHANGED retrieveMetadata(String) public FileMetadata retrieveMetadata(String key) throws IOException try { S3Object object = s3Service.getObjectDetails(bucket, key); return new FileMetadata(key, object.getContentLength(), object.getLastModifiedDate().getTime()); } catch (S3ServiceException e) { // Following is brittle. Is there a better way? if (e.getMessage().contains("ResponseCode=404")) { return null; } handleServiceException(e); // never returned - keep compiler happy return null; }
6018 -1009652553apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 Verify that skip trash option really skips the trash for rmr Verify that skip trash option really skips the trash for rmr FILE_PATH_CHANGED writeFile(FileSystem, Path) protected static Path writeFile(FileSystem fs, Path f) throws IOException DataOutputStream out = fs.create(f); out.writeBytes("dhruba: " + f); out.close(); assertTrue(fs.exists(f)); return f;
6017 -1009652554apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 Verify skip trash option really works Verify skip trash option really works FILE_PATH_CHANGED writeFile(FileSystem, Path) protected static Path writeFile(FileSystem fs, Path f) throws IOException DataOutputStream out = fs.create(f); out.writeBytes("dhruba: " + f); out.close(); assertTrue(fs.exists(f)); return f;
6016 -1009652561apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 verify that after expunging the Trash, it really goes away verify that after expunging the Trash, it really goes away FILE_PATH_CHANGED writeFile(FileSystem, Path) protected static Path writeFile(FileSystem fs, Path f) throws IOException DataOutputStream out = fs.create(f); out.writeBytes("dhruba: " + f); out.close(); assertTrue(fs.exists(f)); return f;
6015 -1009652472apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 Doing a second call with faults disabled should return fine -- ie the internal state of the client or server should not be broken by the failed call Doing a second call with faults disabled should return fine -- ie the internal state of the client or server should not be broken by the failed call FILE_PATH_CHANGED call(Class, Writable, long) public Writable call(Class protocol, Writable param, long receiveTime) throws IOException if (sleep) { // sleep a bit try { Thread.sleep(RANDOM.nextInt(PING_INTERVAL) + MIN_SLEEP_TIME); } catch (InterruptedException e) { } } if (responseClass != null) { try { return responseClass.newInstance(); } catch (Exception e) { throw new RuntimeException(e); } } else { // echo param as result return param; }
6012 -1009652114apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 The following XDR recipe was done through a careful reading of gm_protocol.x in Ganglia 3.1 and carefully examining the output of the gmetric utility with strace. The following XDR recipe was done through a careful reading of gm_protocol.x in Ganglia 3.1 and carefully examining the output of the gmetric utility with strace. FILE_PATH_CHANGED emitMetric(String, String, String, String, GangliaConf, GangliaSlope) protected void emitMetric(String groupName, String name, String type, String value, GangliaConf gConf, GangliaSlope gSlope) throws IOException if (name == null) { LOG.warn("Metric was emitted with no name."); return; } else if (value == null) { LOG.warn("Metric name " + name + " was emitted with a null value."); return; } else if (type == null) { LOG.warn("Metric name " + name + ", value " + value + " has no type."); return; } if (LOG.isDebugEnabled()) { LOG.debug("Emitting metric " + name + ", type " + type + ", value " + value + ", slope " + gSlope.name() + " from hostname " + getHostName()); } // The following XDR recipe was done through a careful reading of // gm_protocol.x in Ganglia 3.1 and carefully examining the output of // the gmetric utility with strace. // First we send out a metadata message // metric_id = metadata_msg xdr_int(128); // hostname xdr_string(getHostName()); // metric name xdr_string(name); // spoof = False xdr_int(0); // metric type xdr_string(type); // metric name xdr_string(name); // units xdr_string(gConf.getUnits()); // slope xdr_int(gSlope.ordinal()); // tmax, the maximum time between metrics xdr_int(gConf.getTmax()); // dmax, the maximum data value xdr_int(gConf.getDmax()); xdr_int(1); /*Num of the entries in extra_value field for Ganglia 3.1.x*/ xdr_string("GROUP"); /*Group attribute*/ xdr_string(groupName); /*Group value*/ // send the metric to Ganglia hosts emitToGangliaHosts(); // Now we send out a message with the actual value. // Technically, we only need to send out the metadata message once for // each metric, but I don't want to have to record which metrics we did and // did not send. // we are sending a string value xdr_int(133); // hostName xdr_string(getHostName()); // metric name xdr_string(name); // spoof = False xdr_int(0); // format field xdr_string("%s"); // metric value xdr_string(value); // send the metric to Ganglia hosts emitToGangliaHosts();
6010 -1009652526apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 The following XDR recipe was done through a careful reading of gm_protocol.x in Ganglia 3.1 and carefully examining the output of the gmetric utility with strace. The following XDR recipe was done through a careful reading of gm_protocol.x in Ganglia 3.1 and carefully examining the output of the gmetric utility with strace. FILE_PATH_CHANGED emitMetric(String, String, String) protected void emitMetric(String name, String type, String value) throws IOException if (name == null) { LOG.warn("Metric was emitted with no name."); return; } else if (value == null) { LOG.warn("Metric name " + name + " was emitted with a null value."); return; } else if (type == null) { LOG.warn("Metric name " + name + ", value " + value + " has no type."); return; } LOG.debug("Emitting metric " + name + ", type " + type + ", value " + value + " from hostname" + hostName); String units = getUnits(name); if (units == null) { LOG.warn("Metric name " + name + ", value " + value + " had 'null' units"); units = ""; } int slope = getSlope(name); int tmax = getTmax(name); int dmax = getDmax(name); offset = 0; String groupName = name.substring(0, name.lastIndexOf(".")); // The following XDR recipe was done through a careful reading of // gm_protocol.x in Ganglia 3.1 and carefully examining the output of // the gmetric utility with strace. // First we send out a metadata message // metric_id = metadata_msg xdr_int(128); // hostname xdr_string(hostName); // metric name xdr_string(name); // spoof = False xdr_int(0); // metric type xdr_string(type); // metric name xdr_string(name); // units xdr_string(units); // slope xdr_int(slope); // tmax, the maximum time between metrics xdr_int(tmax); // dmax, the maximum data value xdr_int(dmax); xdr_int(1); /*Num of the entries in extra_value field for Ganglia 3.1.x*/ xdr_string("GROUP"); /*Group attribute*/ xdr_string(groupName); /*Group value*/ for (SocketAddress socketAddress : metricsServers) { DatagramPacket packet = new DatagramPacket(buffer, offset, socketAddress); datagramSocket.send(packet); } // Now we send out a message with the actual value. // Technically, we only need to send out the metadata message once for // each metric, but I don't want to have to record which metrics we did and // did not send. offset = 0; // we are sending a string value xdr_int(133); // hostName xdr_string(hostName); // metric name xdr_string(name); // spoof = False xdr_int(0); // format field xdr_string("%s"); // metric value xdr_string(value); for (SocketAddress socketAddress : metricsServers) { DatagramPacket packet = new DatagramPacket(buffer, offset, socketAddress); datagramSocket.send(packet); }
6009 -1009652473apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 an artificial way to make a key unique an artificial way to make a key unique FILE_PATH_CHANGED addFileSystemForTesting(URI, Configuration, FileSystem) static void addFileSystemForTesting(URI uri, Configuration conf, FileSystem fs) throws IOException CACHE.map.put(new Cache.Key(uri, conf), fs);
6008 -1009652474apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 Default impl is to assume that permissions do not matter and hence calling the regular mkdirs is good enough. FSs that implement permissions should override this. Default impl is to assume that permissions do not matter and hence calling the regular mkdirs is good enough. FSs that implement permissions should override this. FILE_PATH_CHANGED addFileSystemForTesting(URI, Configuration, FileSystem) static void addFileSystemForTesting(URI uri, Configuration conf, FileSystem fs) throws IOException CACHE.map.put(new Cache.Key(uri, conf), fs);
6007 -1009652475apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 Default impl assumes that permissions do not matter and nor does the bytesPerChecksum hence calling the regular create is good enough. FSs that implement permissions should override this. Default impl assumes that permissions do not matter and nor does the bytesPerChecksum hence calling the regular create is good enough. FSs that implement permissions should override this. FILE_PATH_CHANGED addFileSystemForTesting(URI, Configuration, FileSystem) static void addFileSystemForTesting(URI uri, Configuration conf, FileSystem fs) throws IOException CACHE.map.put(new Cache.Key(uri, conf), fs);
6006 -1009652551apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 The following if clause handles the following case: Assume the following scenario in BZip2 compressed stream where . represent compressed data. .....[48 bit Block].....[48 bit Block].....[48 bit Block]... ........................[47 bits][1 bit].....[48 bit Block]... ................................^[Assume a Byte alignment here] ........................................^^[current position of stream] .....................^^[We go back 10 Bytes in stream and find a Block marker] ........................................^^[We align at wrong position!] ...........................................................^^[While this pos is correct] The following if clause handles the following case: Assume the following scenario in BZip2 compressed stream where . represent compressed data. .....[48 bit Block].....[48 bit Block].....[48 bit Block]... ........................[47 bits][1 bit].....[48 bit Block]... ................................^[Assume a Byte alignment here] ........................................^^[current position of stream] .....................^^[We go back 10 Bytes in stream and find a Block marker] ........................................^^[We align at wrong position!] ...........................................................^^[While this pos is correct] FILE_PATH_CHANGED createOutputStream(OutputStream) public CompressionOutputStream createOutputStream(OutputStream out) throws IOException return new BZip2CompressionOutputStream(out);
6005 -1009652497apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 this is the stupid semantics of FileSystem this is the stupid semantics of FileSystem FILE_PATH_CHANGED mkdirs(Path, FsPermission) public boolean mkdirs(Path dir, FsPermission permission) throws AccessControlException, FileAlreadyExistsException if (theInternalDir.isRoot & dir == null) { throw new FileAlreadyExistsException("/ already exits"); } // Note dir starts with / if (theInternalDir.children.containsKey(dir.toString().substring(1))) { // this is the stupid semantics of FileSystem return true; } throw readOnlyMountTable("mkdirs", dir);
6004 -1009652498apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 The implementors of RawLocalFileSystem were trying to be very smart. They implement FileStatus#getOwener lazily -- the object returned is really a RawLocalFileSystem that expect the FileStatus#getPath to be unchanged so that it can get owner when needed. Hence we need to interpose a new ViewFileSystemFileStatus that works around. The implementors of RawLocalFileSystem were trying to be very smart. They implement FileStatus#getOwener lazily -- the object returned is really a RawLocalFileSystem that expect the FileStatus#getPath to be unchanged so that it can get owner when needed. Hence we need to interpose a new ViewFileSystemFileStatus that works around. FILE_PATH_CHANGED readOnlyMountTable(String, String) static AccessControlException readOnlyMountTable(final String operation, final String p) return new AccessControlException("InternalDir of ViewFileSystem is readonly; operation=" + operation + "Path=" + p);
6003 -1009652523apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 better safe than sorry (should never happen) better safe than sorry (should never happen) FILE_PATH_CHANGED cacheGroupsAdd(List) public void cacheGroupsAdd(List groups) throws IOException for (String group : groups) { if (group.length() == 0) { // better safe than sorry (should never happen) } else if (group.charAt(0) == '@') { if (!NetgroupCache.isCached(group)) { NetgroupCache.add(group, getUsersForNetgroup(group)); } } else { // unix group, not caching } }
6002 -1009652534apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 Benchmark for StringUtils split Benchmark for StringUtils split FILE_PATH_CHANGED main(String[]) public static void main(String[] args) final String TO_SPLIT = "foo,bar,baz,blah,blah"; for (boolean useOurs : new boolean[] { false, true }) { for (int outer = 0; outer < 10; outer++) { long st = System.nanoTime(); int components = 0; for (int inner = 0; inner < 1000000; inner++) { String[] res; if (useOurs) { res = StringUtils.split(TO_SPLIT, ','); } else { res = TO_SPLIT.split(","); } // be sure to use res, otherwise might be optimized out components += res.length; } long et = System.nanoTime(); if (outer > 3) { System.out.println((useOurs ? "StringUtils impl" : "Java impl") + " #" + outer + ":" + (et - st) / 1000000 + "ms"); } } }
5999 -1009652610apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 Ugly utility, maybe someone else can do this better Ugly utility, maybe someone else can do this better FILE_PATH_CHANGED readCompressedString(DataInput) public static String readCompressedString(DataInput in) throws IOException byte[] bytes = readCompressedByteArray(in); if (bytes == null) return null; return new String(bytes, "UTF-8");
5997 -1009652485apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 TODO: should probably allow path arguments for the filesystems TODO: should probably allow path arguments for the filesystems FILE_PATH_CHANGED processOptions(LinkedList) protected void processOptions(LinkedList args) throws IOException CommandFormat cf = new CommandFormat(0, 0); cf.parse(args);
5996 -1009652486apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 TODO: if the user wants the trash to be used but there is any problem (ie. creating the trash dir, moving the item to be deleted, etc), then the path will just be deleted because moveToTrash returns false and it falls thru to fs.delete. this doesn't seem right TODO: if the user wants the trash to be used but there is any problem (ie. creating the trash dir, moving the item to be deleted, etc), then the path will just be deleted because moveToTrash returns false and it falls thru to fs.delete. this doesn't seem right FILE_PATH_CHANGED registerCommands(CommandFactory) public static void registerCommands(CommandFactory factory) factory.addClass(Rm.class, "-rm"); factory.addClass(Rmdir.class, "-rmdir"); factory.addClass(Rmr.class, "-rmr"); factory.addClass(Expunge.class, "-expunge");
5995 -1009652520apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 For efficient implementation, there's no way around the following massive code duplication. For efficient implementation, there's no way around the following massive code duplication. FILE_PATH_CHANGED writeBooleanArray(DataOutput) private void writeBooleanArray(DataOutput out) throws IOException boolean[] v = (boolean[]) value; for (int i = 0; i < length; i++) out.writeBoolean(v[i]);
5994 -1009652538apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 Flip to the newly parsed permissions Flip to the newly parsed permissions FILE_PATH_CHANGED refresh(Configuration, PolicyProvider) public synchronized void refresh(Configuration conf, PolicyProvider provider) // Get the system property 'hadoop.policy.file' String policyFile = System.getProperty("hadoop.policy.file", HADOOP_POLICY_FILE); // Make a copy of the original config, and load the policy file Configuration policyConf = new Configuration(conf); policyConf.addResource(policyFile); final Map, AccessControlList> newAcls = new IdentityHashMap, AccessControlList>(); // Parse the config file Service[] services = provider.getServices(); if (services != null) { for (Service service : services) { AccessControlList acl = new AccessControlList(policyConf.get(service.getServiceKey(), AccessControlList.WILDCARD_ACL_VALUE)); newAcls.put(service.getProtocol(), acl); } } // Flip to the newly parsed permissions protocolToAcl = newAcls;
5993 -1009652611apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 Read characters into a portion of an array, reading from the underlying stream at most once if necessary. Read characters into a portion of an array, reading from the underlying stream at most once if necessary. FILE_PATH_CHANGED read1(byte[], int, int) private int read1(byte[] b, int off, int len) throws IOException int avail = count - pos; if (avail <= 0) { if (len >= maxChunkSize) { // read a chunk to user buffer directly; avoid one copy int nread = readChecksumChunk(b, off, len); return nread; } else { // read a chunk into the local buffer fill(); if (count <= 0) { return -1; } else { avail = count; } } } // copy content of the local buffer to the user buffer int cnt = (avail < len) ? avail : len; System.arraycopy(buf, pos, b, off, cnt); pos += cnt; return cnt;
5991 -1009652530apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 FIXME? Inflater docs say: 'it is also necessary to provide an extra \"dummy\" byte as input. This is required by the ZLIB native library in order to support certain optimizations.' However, this does not appear to be true, and in any case, it's not entirely clear where the byte should go or what its value should be. Perhaps it suffices to have some deflated bytes in the first buffer load? (But how else would one do it?) FIXME? Inflater docs say: 'it is also necessary to provide an extra \"dummy\" byte as input. This is required by the ZLIB native library in order to support certain optimizations.' However, this does not appear to be true, and in any case, it's not entirely clear where the byte should go or what its value should be. Perhaps it suffices to have some deflated bytes in the first buffer load? (But how else would one do it?) FILE_PATH_CHANGED needsInput() public synchronized boolean needsInput() if (state == GzipStateLabel.DEFLATE_STREAM) { // most common case return inflater.needsInput(); } // see userBufLen comment at top of decompress(); currently no need to // verify userBufLen <= 0 return (state != GzipStateLabel.FINISHED);
5990 -1009652510apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 this is an unexpected condition, so dump the whole exception since it's probably a nasty internal error where the backtrace would be useful this is an unexpected condition, so dump the whole exception since it's probably a nasty internal error where the backtrace would be useful FILE_PATH_CHANGED displayError(Exception) public void displayError(Exception e) // build up a list of exceptions that occurred exceptions.add(e); String errorMessage = e.getLocalizedMessage(); if (errorMessage == null) { // this is an unexpected condition, so dump the whole exception since // it's probably a nasty internal error where the backtrace would be // useful errorMessage = StringUtils.stringifyException(e); LOG.debug(errorMessage); } else { errorMessage = errorMessage.split("\n", 2)[0]; } displayError(errorMessage);
5989 -1009652512apache/hadoopArun Murthy0f6dfeeacbab65a31a33927a4eb84871d371fe52 TODO: this really should be iterative TODO: this really should be iterative FILE_PATH_CHANGED processPaths(PathData, PathData...) protected void processPaths(PathData parent, PathData... items) throws IOException // TODO: this really should be iterative for (PathData item : items) { try { processPath(item); if (recursive && item.stat.isDirectory()) { recursePath(item); } } catch (IOException e) { displayError(e); } }
5988 -1009651780apache/hadoopSteve Loughran28e6a4e44a3e920dcaf858f9a74a6358226b3a63 None Calling recover should move it aside SATD_ADDED testLogGroupRecoveryInProgress() public void testLogGroupRecoveryInProgress() throws IOException String[] paths = new String[] { "/foo1/current/" + getInProgressEditsFileName(123), "/foo2/current/" + getInProgressEditsFileName(123), "/foo3/current/" + getInProgressEditsFileName(123) }; FSImageTransactionalStorageInspector inspector = new FSImageTransactionalStorageInspector(); inspector.inspectDirectory(mockDirectoryWithEditLogs(paths[0])); inspector.inspectDirectory(mockDirectoryWithEditLogs(paths[1])); inspector.inspectDirectory(mockDirectoryWithEditLogs(paths[2])); // Inject spies to return the valid counts we would like to see mockLogValidation(inspector, paths[0], 2000); mockLogValidation(inspector, paths[1], 2000); mockLogValidation(inspector, paths[2], 1000); LogGroup lg = inspector.logGroups.get(123L); assertEquals(3, lg.logs.size()); lg.planRecovery(); // Check that the short one was marked corrupt assertFalse(lg.logs.get(0).isCorrupt()); assertFalse(lg.logs.get(1).isCorrupt()); assertTrue(lg.logs.get(2).isCorrupt()); // Calling recover should move it aside FoundEditLog badLog = lg.logs.get(2); Mockito.doNothing().when(badLog).moveAsideCorruptFile(); Mockito.doNothing().when(lg.logs.get(0)).finalizeLog(); Mockito.doNothing().when(lg.logs.get(1)).finalizeLog(); lg.recover(); Mockito.verify(badLog).moveAsideCorruptFile(); Mockito.verify(lg.logs.get(0)).finalizeLog(); Mockito.verify(lg.logs.get(1)).finalizeLog();
5987 -1009651781apache/hadoopSteve Loughran28e6a4e44a3e920dcaf858f9a74a6358226b3a63 None Calling recover should move it aside SATD_ADDED testLogGroupRecoveryMixed() public void testLogGroupRecoveryMixed() throws IOException FSImageTransactionalStorageInspector inspector = new FSImageTransactionalStorageInspector(); inspector.inspectDirectory(mockDirectoryWithEditLogs("/foo1/current/" + getFinalizedEditsFileName(123, 456))); inspector.inspectDirectory(mockDirectoryWithEditLogs("/foo2/current/" + getFinalizedEditsFileName(123, 456))); inspector.inspectDirectory(mockDirectoryWithEditLogs("/foo3/current/" + getInProgressEditsFileName(123))); inspector.inspectDirectory(mockDirectory(NameNodeDirType.IMAGE, false, "/foo4/current/" + getImageFileName(122))); LogGroup lg = inspector.logGroups.get(123L); assertEquals(3, lg.logs.size()); FoundEditLog inProgressLog = lg.logs.get(2); assertTrue(inProgressLog.isInProgress()); LoadPlan plan = inspector.createLoadPlan(); // Check that it was marked corrupt. assertFalse(lg.logs.get(0).isCorrupt()); assertFalse(lg.logs.get(1).isCorrupt()); assertTrue(lg.logs.get(2).isCorrupt()); // Calling recover should move it aside inProgressLog = spy(inProgressLog); Mockito.doNothing().when(inProgressLog).moveAsideCorruptFile(); lg.logs.set(2, inProgressLog); plan.doRecovery(); Mockito.verify(inProgressLog).moveAsideCorruptFile();
5986 -1009651782apache/hadoopSteve Loughran28e6a4e44a3e920dcaf858f9a74a6358226b3a63 None Corrupt the log file in various ways for each txn SATD_ADDED testCountValidTransactions() public void testCountValidTransactions() throws IOException File testDir = new File(TEST_DIR, "testCountValidTransactions"); File logFile = new File(testDir, NNStorage.getInProgressEditsFileName(1)); // Create a log file, and return the offsets at which each // transaction starts. FSEditLog fsel = null; final int NUM_TXNS = 30; SortedMap offsetToTxId = Maps.newTreeMap(); try { fsel = FSImageTestUtil.createStandaloneEditLog(testDir); fsel.open(); assertTrue("should exist: " + logFile, logFile.exists()); for (int i = 0; i < NUM_TXNS; i++) { long trueOffset = getNonTrailerLength(logFile); long thisTxId = fsel.getLastWrittenTxId() + 1; offsetToTxId.put(trueOffset, thisTxId); System.err.println("txid " + thisTxId + " at offset " + trueOffset); fsel.logDelete("path" + i, i); fsel.logSync(); } } finally { if (fsel != null) { fsel.close(); } } // The file got renamed when the log was closed. logFile = testDir.listFiles()[0]; long validLength = getNonTrailerLength(logFile); // Make sure that uncorrupted log has the expected length and number // of transactions. EditLogValidation validation = FSEditLogLoader.validateEditLog(logFile); assertEquals(NUM_TXNS + 2, validation.numTransactions); assertEquals(validLength, validation.validLength); // Back up the uncorrupted log File logFileBak = new File(testDir, logFile.getName() + ".bak"); Files.copy(logFile, logFileBak); // Corrupt the log file in various ways for each txn for (Map.Entry entry : offsetToTxId.entrySet()) { long txOffset = entry.getKey(); long txid = entry.getValue(); // Restore backup, truncate the file exactly before the txn Files.copy(logFileBak, logFile); truncateFile(logFile, txOffset); validation = FSEditLogLoader.validateEditLog(logFile); assertEquals("Failed when truncating to length " + txOffset, txid - 1, validation.numTransactions); assertEquals(txOffset, validation.validLength); // Restore backup, truncate the file with one byte in the txn, // also isn't valid Files.copy(logFileBak, logFile); truncateFile(logFile, txOffset + 1); validation = FSEditLogLoader.validateEditLog(logFile); assertEquals("Failed when truncating to length " + (txOffset + 1), txid - 1, validation.numTransactions); assertEquals(txOffset, validation.validLength); // Restore backup, corrupt the txn opcode Files.copy(logFileBak, logFile); corruptByteInFile(logFile, txOffset); validation = FSEditLogLoader.validateEditLog(logFile); assertEquals("Failed when corrupting txn opcode at " + txOffset, txid - 1, validation.numTransactions); assertEquals(txOffset, validation.validLength); // Restore backup, corrupt a byte a few bytes into the txn Files.copy(logFileBak, logFile); corruptByteInFile(logFile, txOffset + 5); validation = FSEditLogLoader.validateEditLog(logFile); assertEquals("Failed when corrupting txn data at " + (txOffset + 5), txid - 1, validation.numTransactions); assertEquals(txOffset, validation.validLength); } // Corrupt the log at every offset to make sure that validation itself // never throws an exception, and that the calculated lengths are monotonically // increasing long prevNumValid = 0; for (long offset = 0; offset < validLength; offset++) { Files.copy(logFileBak, logFile); corruptByteInFile(logFile, offset); EditLogValidation val = FSEditLogLoader.validateEditLog(logFile); assertTrue(val.numTransactions >= prevNumValid); prevNumValid = val.numTransactions; }
5985 -1009651783apache/hadoopSteve Loughran28e6a4e44a3e920dcaf858f9a74a6358226b3a63 None TODO: unfortunately this fails -- should be improved. See HDFS-2173. SATD_ADDED saveNamespaceWithInjectedFault(Fault) private void saveNamespaceWithInjectedFault(Fault fault) throws Exception Configuration conf = getConf(); NameNode.initMetrics(conf, NamenodeRole.NAMENODE); DFSTestUtil.formatNameNode(conf); FSNamesystem fsn = new FSNamesystem(conf); // Replace the FSImage with a spy FSImage originalImage = fsn.dir.fsImage; NNStorage storage = originalImage.getStorage(); NNStorage spyStorage = spy(storage); originalImage.storage = spyStorage; FSImage spyImage = spy(originalImage); fsn.dir.fsImage = spyImage; // should we expect the save operation to fail boolean shouldFail = false; // inject fault switch(fault) { case SAVE_SECOND_FSIMAGE_RTE: // The spy throws a RuntimeException when writing to the second directory doAnswer(new FaultySaveImage(true)).when(spyImage).saveFSImage((StorageDirectory) anyObject(), anyLong()); shouldFail = false; break; case SAVE_SECOND_FSIMAGE_IOE: // The spy throws an IOException when writing to the second directory doAnswer(new FaultySaveImage(false)).when(spyImage).saveFSImage((StorageDirectory) anyObject(), anyLong()); shouldFail = false; break; case SAVE_ALL_FSIMAGES: // The spy throws IOException in all directories doThrow(new RuntimeException("Injected")).when(spyImage).saveFSImage((StorageDirectory) anyObject(), anyLong()); shouldFail = true; break; case WRITE_STORAGE_ALL: // The spy throws an exception before writing any VERSION files doThrow(new RuntimeException("Injected")).when(spyStorage).writeAll(); shouldFail = true; break; case WRITE_STORAGE_ONE: // The spy throws on exception on one particular storage directory doAnswer(new FaultySaveImage(true)).when(spyStorage).writeProperties((StorageDirectory) anyObject()); // TODO: unfortunately this fails -- should be improved. // See HDFS-2173. shouldFail = true; break; } try { doAnEdit(fsn, 1); // Save namespace - this may fail, depending on fault injected fsn.setSafeMode(SafeModeAction.SAFEMODE_ENTER); try { fsn.saveNamespace(); if (shouldFail) { fail("Did not fail!"); } } catch (Exception e) { if (!shouldFail) { throw e; } else { LOG.info("Test caught expected exception", e); } } fsn.setSafeMode(SafeModeAction.SAFEMODE_LEAVE); // Should still be able to perform edits doAnEdit(fsn, 2); // Now shut down and restart the namesystem originalImage.close(); fsn.close(); fsn = null; // Start a new namesystem, which should be able to recover // the namespace from the previous incarnation. fsn = new FSNamesystem(conf); // Make sure the image loaded including our edits. checkEditExists(fsn, 1); checkEditExists(fsn, 2); } finally { if (fsn != null) { fsn.close(); } }
5984 -1009651784apache/hadoopSteve Loughran28e6a4e44a3e920dcaf858f9a74a6358226b3a63 None TODO writeRaw SATD_ADDED initialValue() protected Checksum initialValue() return new PureJavaCrc32();
5983 -1009651785apache/hadoopSteve Loughran28e6a4e44a3e920dcaf858f9a74a6358226b3a63 None TODO no need to link this back to storage anymore! See HDFS-2174. SATD_ADDED initialValue() protected Checksum initialValue() return new PureJavaCrc32();
5982 -1009651786apache/hadoopSteve Loughran28e6a4e44a3e920dcaf858f9a74a6358226b3a63 None We appear to have missed some transactions -- the NN probably lost contact with us temporarily. So, mark the current segment as aborted. SATD_ADDED namenodeStartedLogSegment(long) LOG.info("NameNode started a new log segment at txid " + txid); if (editLog.isOpen()) { if (editLog.getLastWrittenTxId() == txid - 1) { // We are in sync with the NN, so end and finalize the current segment editLog.endCurrentLogSegment(false); } else { // We appear to have missed some transactions -- the NN probably // lost contact with us temporarily. So, mark the current segment // as aborted. LOG.warn("NN started new log segment at txid " + txid + ", but BN had only written up to txid " + editLog.getLastWrittenTxId() + "in the log segment starting at " + editLog.getCurSegmentTxId() + ". Aborting this " + "log segment."); editLog.abortCurrentLogSegment(); } } editLog.setNextTxId(txid); editLog.startLogSegment(txid, false); if (bnState == BNState.DROP_UNTIL_NEXT_ROLL) { setState(BNState.JOURNAL_ONLY); } if (stopApplyingEditsOnNextRoll) { if (bnState == BNState.IN_SYNC) { LOG.info("Stopped applying edits to prepare for checkpoint."); setState(BNState.JOURNAL_ONLY); } stopApplyingEditsOnNextRoll = false; notifyAll(); } synchronized void namenodeStartedLogSegment(long txid) throws IOException
5981 -1009651787apache/hadoopSteve Loughran28e6a4e44a3e920dcaf858f9a74a6358226b3a63 None this doesn't directly upload an image, but rather asks the NN to connect back to the 2NN to download the specified image. SATD_ADDED uploadImageFromStorage(String, InetSocketAddress, NNStorage, long) static void uploadImageFromStorage(String fsName, InetSocketAddress imageListenAddress, NNStorage storage, long txid) throws IOException String fileid = GetImageServlet.getParamStringToPutImage(txid, imageListenAddress, storage); // this doesn't directly upload an image, but rather asks the NN // to connect back to the 2NN to download the specified image. TransferFsImage.getFileClient(fsName, fileid, null, null, false); LOG.info("Uploaded image with txid " + txid + " to namenode at " + fsName);
5980 -1009652425apache/hadoopSteve Loughran28e6a4e44a3e920dcaf858f9a74a6358226b3a63 This really shouldn't happen... This really shouldn't happen... CLASS_OR_METHOD_CHANGED doGet(HttpServletRequest, HttpServletResponse) public void doGet(final HttpServletRequest request, final HttpServletResponse response) throws ServletException, IOException try { ServletContext context = getServletContext(); final FSImage nnImage = NameNodeHttpServer.getFsImageFromContext(context); final GetImageParams parsedParams = new GetImageParams(request, response); final Configuration conf = (Configuration) getServletContext().getAttribute(JspHelper.CURRENT_CONF); if (UserGroupInformation.isSecurityEnabled() && !isValidRequestor(request.getRemoteUser(), conf)) { response.sendError(HttpServletResponse.SC_FORBIDDEN, "Only Namenode and Secondary Namenode may access this servlet"); LOG.warn("Received non-NN/SNN request for image or edits from " + request.getRemoteHost()); return; } String myStorageInfoString = nnImage.getStorage().toColonSeparatedString(); String theirStorageInfoString = parsedParams.getStorageInfoString(); if (theirStorageInfoString != null && !myStorageInfoString.equals(theirStorageInfoString)) { response.sendError(HttpServletResponse.SC_FORBIDDEN, "This namenode has storage info " + myStorageInfoString + " but the secondary expected " + theirStorageInfoString); LOG.warn("Received an invalid request file transfer request " + "from a secondary with storage info " + theirStorageInfoString); return; } UserGroupInformation.getCurrentUser().doAs(new PrivilegedExceptionAction() { @Override public Void run() throws Exception { if (parsedParams.isGetImage()) { long txid = parsedParams.getTxId(); File imageFile = nnImage.getStorage().getFsImageName(txid); if (imageFile == null) { throw new IOException("Could not find image with txid " + txid); } setVerificationHeaders(response, imageFile); // send fsImage TransferFsImage.getFileServer(response.getOutputStream(), imageFile, getThrottler(conf)); } else if (parsedParams.isGetEdit()) { long startTxId = parsedParams.getStartTxId(); long endTxId = parsedParams.getEndTxId(); File editFile = nnImage.getStorage().findFinalizedEditsFile(startTxId, endTxId); setVerificationHeaders(response, editFile); // send edits TransferFsImage.getFileServer(response.getOutputStream(), editFile, getThrottler(conf)); } else if (parsedParams.isPutImage()) { final long txid = parsedParams.getTxId(); if (!currentlyDownloadingCheckpoints.add(txid)) { throw new IOException("Another checkpointer is already in the process of uploading a" + " checkpoint made at transaction ID " + txid); } try { if (nnImage.getStorage().findImageFile(txid) != null) { throw new IOException("Another checkpointer already uploaded an checkpoint " + "for txid " + txid); } // issue a HTTP get request to download the new fsimage MD5Hash downloadImageDigest = reloginIfNecessary().doAs(new PrivilegedExceptionAction() { @Override public MD5Hash run() throws Exception { return TransferFsImage.downloadImageToStorage(parsedParams.getInfoServer(), txid, nnImage.getStorage(), true); } }); nnImage.saveDigestAndRenameCheckpointImage(txid, downloadImageDigest); // Now that we have a new checkpoint, we might be able to // remove some old ones. nnImage.purgeOldStorage(); } finally { currentlyDownloadingCheckpoints.remove(txid); } } return null; } // We may have lost our ticket since the last time we tried to open // an http connection, so log in just in case. private UserGroupInformation reloginIfNecessary() throws IOException { // This method is only called on the NN, therefore it is safe to // use these key values. return UserGroupInformation.loginUserFromKeytabAndReturnUGI(SecurityUtil.getServerPrincipal(conf.get(DFSConfigKeys.DFS_NAMENODE_KRB_HTTPS_USER_NAME_KEY), NameNode.getAddress(conf).getHostName()), conf.get(DFSConfigKeys.DFS_NAMENODE_KEYTAB_FILE_KEY)); } }); } catch (Exception ie) { String errMsg = "GetImage failed. " + StringUtils.stringifyException(ie); response.sendError(HttpServletResponse.SC_GONE, errMsg); throw new IOException(errMsg); } finally { response.getOutputStream().close(); }
5978 -1009652211apache/hadoopSteve Loughran28e6a4e44a3e920dcaf858f9a74a6358226b3a63 Should have incremented the batch count exactly once Should have incremented the batch count exactly once CLASS_OR_METHOD_CHANGED run() public void run() PermissionStatus p = namesystem.createFsOwnerPermissions(new FsPermission((short) 0777)); FSEditLog editLog = namesystem.getEditLog(); for (int i = 0; i < numTransactions; i++) { INodeFileUnderConstruction inode = new INodeFileUnderConstruction(p, replication, blockSize, 0, "", "", null); editLog.logOpenFile("/filename" + i, inode); editLog.logCloseFile("/filename" + i, inode); editLog.logSync(); }
5977 -1009652357apache/hadoopSteve Loughran28e6a4e44a3e920dcaf858f9a74a6358226b3a63 verify that 'format' really blew away all pre-existing files verify that 'format' really blew away all pre-existing files CLASS_OR_METHOD_CHANGED setUp() public void setUp() throws IOException FileUtil.fullyDeleteContents(new File(MiniDFSCluster.getBaseDirectory())); ErrorSimulator.initializeErrorSimulationEvent(5);
5976 -1009652358apache/hadoopSteve Loughran28e6a4e44a3e920dcaf858f9a74a6358226b3a63 We should probably test for more of the file properties. We should probably test for more of the file properties. CLASS_OR_METHOD_CHANGED setUp() public void setUp() throws IOException FileUtil.fullyDeleteContents(new File(MiniDFSCluster.getBaseDirectory())); ErrorSimulator.initializeErrorSimulationEvent(5);
5975 -1009652438apache/hadoopSteve Loughran28e6a4e44a3e920dcaf858f9a74a6358226b3a63 TODO bad dependency TODO bad dependency CLASS_OR_METHOD_CHANGED save(File, long, FSNamesystem, FSImageCompression) void save(File newFile, long txid, FSNamesystem sourceNamesystem, FSImageCompression compression) throws IOException checkNotSaved(); FSDirectory fsDir = sourceNamesystem.dir; long startTime = now(); // // Write out data // MessageDigest digester = MD5Hash.getDigester(); FileOutputStream fout = new FileOutputStream(newFile); DigestOutputStream fos = new DigestOutputStream(fout, digester); DataOutputStream out = new DataOutputStream(fos); try { out.writeInt(FSConstants.LAYOUT_VERSION); out.writeInt(sourceNamesystem.getFSImage().getStorage().getNamespaceID()); out.writeLong(fsDir.rootDir.numItemsInTree()); out.writeLong(sourceNamesystem.getGenerationStamp()); out.writeLong(txid); // write compression info and set up compressed stream out = compression.writeHeaderAndWrapStream(fos); LOG.info("Saving image file " + newFile + " using " + compression); byte[] byteStore = new byte[4 * FSConstants.MAX_PATH_LENGTH]; ByteBuffer strbuf = ByteBuffer.wrap(byteStore); // save the root FSImageSerialization.saveINode2Image(fsDir.rootDir, out); // save the rest of the nodes saveImage(strbuf, fsDir.rootDir, out); // save files under construction sourceNamesystem.saveFilesUnderConstruction(out); sourceNamesystem.saveSecretManagerState(out); strbuf = null; out.flush(); fout.getChannel().force(true); } finally { out.close(); } saved = true; // set md5 of the saved image savedDigest = new MD5Hash(digester.digest()); LOG.info("Image file of size " + newFile.length() + " saved in " + (now() - startTime) / 1000 + " seconds.");
5974 -1009652388apache/hadoopSteve Loughran28e6a4e44a3e920dcaf858f9a74a6358226b3a63 Poll the Namenode (once every 5 minutes) to find the size of the pending edit log. 5 minutes Poll the Namenode (once every checkpointCheckPeriod seconds) to find the number of transactions in the edit log that haven't yet been checkpointed. SATD_REMOVED toString() public String toString() return getClass().getSimpleName() + " Status" + "\nName Node Address : " + nameNodeAddr + "\nStart Time : " + new Date(starttime) + "\nLast Checkpoint Time : " + (lastCheckpointTime == 0 ? "--" : new Date(lastCheckpointTime)) + "\nCheckpoint Period : " + checkpointPeriod + " seconds" + "\nCheckpoint Size : " + StringUtils.byteDesc(checkpointTxnCount) + " (= " + checkpointTxnCount + " bytes)" + "\nCheckpoint Dirs : " + checkpointDirs + "\nCheckpoint Edits Dirs: " + checkpointEditsDirs;
5973 -1009651788apache/hadoopTsz-wo Sze184ff33de5598b04bf968ea3b113b175a4225b82 move to current trash move to current trash SATD_MOVED_FILE initialize(Configuration, FileSystem, Path) public void initialize(Configuration conf, FileSystem fs, Path home) this.fs = fs; this.trash = new Path(home, TRASH); this.homesParent = home.getParent(); this.current = new Path(trash, CURRENT); this.deletionInterval = (long) (conf.getFloat(FS_TRASH_INTERVAL_KEY, FS_TRASH_INTERVAL_DEFAULT) * MSECS_PER_MINUTE);
5712 -1009651936apache/hadoopTsz-wo Szecabec474e0f4c525ba80bd7c8dede1a8d76e4f39 // verify policy deletes the correct blocks. companion blocks should be // evenly distributed. create a new BlockPlacementPolicyRaid to clear the cache SATD_REMOVED refreshPolicy() private void refreshPolicy() policy = new BlockPlacementPolicyRaid(); policy.initialize(conf, namesystem, namesystem.clusterMap);
5710 -1009651937apache/hadoopTsz-wo Szecabec474e0f4c525ba80bd7c8dede1a8d76e4f39 // verify policy deletes the correct blocks. companion blocks should be // evenly distributed. verify policy deletes the correct blocks. companion blocks should be evenly distributed. SATD_CHANGED testDeleteReplica() public void testDeleteReplica() throws IOException setupCluster(); try { // Set the policy to default policy to place the block in the default way setBlockPlacementPolicy(namesystem, new BlockPlacementPolicyDefault(conf, namesystem, namesystem.clusterMap)); DatanodeDescriptor datanode1 = NameNodeRaidTestUtil.getDatanodeMap(namesystem).values().iterator().next(); String source = "/dir/file"; String parity = xorPrefix + source; final Path parityPath = new Path(parity); DFSTestUtil.createFile(fs, parityPath, 3, (short) 1, 0L); DFSTestUtil.waitReplication(fs, parityPath, (short) 1); // start one more datanode cluster.startDataNodes(conf, 1, true, null, rack2, host2, null); DatanodeDescriptor datanode2 = null; for (DatanodeDescriptor d : NameNodeRaidTestUtil.getDatanodeMap(namesystem).values()) { if (!d.getName().equals(datanode1.getName())) { datanode2 = d; } } Assert.assertTrue(datanode2 != null); cluster.waitActive(); final Path sourcePath = new Path(source); DFSTestUtil.createFile(fs, sourcePath, 5, (short) 2, 0L); DFSTestUtil.waitReplication(fs, sourcePath, (short) 2); refreshPolicy(); Assert.assertEquals(parity, policy.getParityFile(source)); Assert.assertEquals(source, policy.getSourceFile(parity, xorPrefix)); List sourceBlocks = getBlocks(namesystem, source); List parityBlocks = getBlocks(namesystem, parity); Assert.assertEquals(5, sourceBlocks.size()); Assert.assertEquals(3, parityBlocks.size()); // verify the result of getCompanionBlocks() Collection companionBlocks; companionBlocks = getCompanionBlocks(namesystem, policy, sourceBlocks.get(0).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 0, 1 }, new int[] { 0 }); companionBlocks = getCompanionBlocks(namesystem, policy, sourceBlocks.get(1).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 0, 1 }, new int[] { 0 }); companionBlocks = getCompanionBlocks(namesystem, policy, sourceBlocks.get(2).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 2, 3 }, new int[] { 1 }); companionBlocks = getCompanionBlocks(namesystem, policy, sourceBlocks.get(3).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 2, 3 }, new int[] { 1 }); companionBlocks = getCompanionBlocks(namesystem, policy, sourceBlocks.get(4).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 4 }, new int[] { 2 }); companionBlocks = getCompanionBlocks(namesystem, policy, parityBlocks.get(0).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 0, 1 }, new int[] { 0 }); companionBlocks = getCompanionBlocks(namesystem, policy, parityBlocks.get(1).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 2, 3 }, new int[] { 1 }); companionBlocks = getCompanionBlocks(namesystem, policy, parityBlocks.get(2).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 4 }, new int[] { 2 }); // Set the policy back to raid policy. We have to create a new object // here to clear the block location cache refreshPolicy(); setBlockPlacementPolicy(namesystem, policy); // verify policy deletes the correct blocks. companion blocks should be // evenly distributed. fs.setReplication(sourcePath, (short) 1); DFSTestUtil.waitReplication(fs, sourcePath, (short) 1); Map counters = new HashMap(); refreshPolicy(); for (int i = 0; i < parityBlocks.size(); i++) { companionBlocks = getCompanionBlocks(namesystem, policy, parityBlocks.get(i).getBlock()); counters = BlockPlacementPolicyRaid.countCompanionBlocks(companionBlocks, false); Assert.assertTrue(counters.get(datanode1.getName()) >= 1 && counters.get(datanode1.getName()) <= 2); Assert.assertTrue(counters.get(datanode1.getName()) + counters.get(datanode2.getName()) == companionBlocks.size()); counters = BlockPlacementPolicyRaid.countCompanionBlocks(companionBlocks, true); Assert.assertTrue(counters.get(datanode1.getParent().getName()) >= 1 && counters.get(datanode1.getParent().getName()) <= 2); Assert.assertTrue(counters.get(datanode1.getParent().getName()) + counters.get(datanode2.getParent().getName()) == companionBlocks.size()); } } finally { if (cluster != null) { cluster.shutdown(); } }
5708 -1009651938apache/hadoopTsz-wo Szecabec474e0f4c525ba80bd7c8dede1a8d76e4f39 // verify policy deletes the correct blocks. companion blocks should be // evenly distributed. Set the policy back to raid policy. We have to create a new object here to clear the block location cache SATD_REMOVED testDeleteReplica() public void testDeleteReplica() throws IOException setupCluster(); try { // Set the policy to default policy to place the block in the default way setBlockPlacementPolicy(namesystem, new BlockPlacementPolicyDefault(conf, namesystem, namesystem.clusterMap)); DatanodeDescriptor datanode1 = NameNodeRaidTestUtil.getDatanodeMap(namesystem).values().iterator().next(); String source = "/dir/file"; String parity = xorPrefix + source; final Path parityPath = new Path(parity); DFSTestUtil.createFile(fs, parityPath, 3, (short) 1, 0L); DFSTestUtil.waitReplication(fs, parityPath, (short) 1); // start one more datanode cluster.startDataNodes(conf, 1, true, null, rack2, host2, null); DatanodeDescriptor datanode2 = null; for (DatanodeDescriptor d : NameNodeRaidTestUtil.getDatanodeMap(namesystem).values()) { if (!d.getName().equals(datanode1.getName())) { datanode2 = d; } } Assert.assertTrue(datanode2 != null); cluster.waitActive(); final Path sourcePath = new Path(source); DFSTestUtil.createFile(fs, sourcePath, 5, (short) 2, 0L); DFSTestUtil.waitReplication(fs, sourcePath, (short) 2); refreshPolicy(); Assert.assertEquals(parity, policy.getParityFile(source)); Assert.assertEquals(source, policy.getSourceFile(parity, xorPrefix)); List sourceBlocks = getBlocks(namesystem, source); List parityBlocks = getBlocks(namesystem, parity); Assert.assertEquals(5, sourceBlocks.size()); Assert.assertEquals(3, parityBlocks.size()); // verify the result of getCompanionBlocks() Collection companionBlocks; companionBlocks = getCompanionBlocks(namesystem, policy, sourceBlocks.get(0).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 0, 1 }, new int[] { 0 }); companionBlocks = getCompanionBlocks(namesystem, policy, sourceBlocks.get(1).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 0, 1 }, new int[] { 0 }); companionBlocks = getCompanionBlocks(namesystem, policy, sourceBlocks.get(2).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 2, 3 }, new int[] { 1 }); companionBlocks = getCompanionBlocks(namesystem, policy, sourceBlocks.get(3).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 2, 3 }, new int[] { 1 }); companionBlocks = getCompanionBlocks(namesystem, policy, sourceBlocks.get(4).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 4 }, new int[] { 2 }); companionBlocks = getCompanionBlocks(namesystem, policy, parityBlocks.get(0).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 0, 1 }, new int[] { 0 }); companionBlocks = getCompanionBlocks(namesystem, policy, parityBlocks.get(1).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 2, 3 }, new int[] { 1 }); companionBlocks = getCompanionBlocks(namesystem, policy, parityBlocks.get(2).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 4 }, new int[] { 2 }); // Set the policy back to raid policy. We have to create a new object // here to clear the block location cache refreshPolicy(); setBlockPlacementPolicy(namesystem, policy); // verify policy deletes the correct blocks. companion blocks should be // evenly distributed. fs.setReplication(sourcePath, (short) 1); DFSTestUtil.waitReplication(fs, sourcePath, (short) 1); Map counters = new HashMap(); refreshPolicy(); for (int i = 0; i < parityBlocks.size(); i++) { companionBlocks = getCompanionBlocks(namesystem, policy, parityBlocks.get(i).getBlock()); counters = BlockPlacementPolicyRaid.countCompanionBlocks(companionBlocks, false); Assert.assertTrue(counters.get(datanode1.getName()) >= 1 && counters.get(datanode1.getName()) <= 2); Assert.assertTrue(counters.get(datanode1.getName()) + counters.get(datanode2.getName()) == companionBlocks.size()); counters = BlockPlacementPolicyRaid.countCompanionBlocks(companionBlocks, true); Assert.assertTrue(counters.get(datanode1.getParent().getName()) >= 1 && counters.get(datanode1.getParent().getName()) <= 2); Assert.assertTrue(counters.get(datanode1.getParent().getName()) + counters.get(datanode2.getParent().getName()) == companionBlocks.size()); } } finally { if (cluster != null) { cluster.shutdown(); } }
5707 -1009651939apache/hadoopTsz-wo Szecabec474e0f4c525ba80bd7c8dede1a8d76e4f39 // verify policy deletes the correct blocks. companion blocks should be // evenly distributed. verify the result of getCompanionBlocks() SATD_REMOVED testDeleteReplica() public void testDeleteReplica() throws IOException setupCluster(); try { // Set the policy to default policy to place the block in the default way setBlockPlacementPolicy(namesystem, new BlockPlacementPolicyDefault(conf, namesystem, namesystem.clusterMap)); DatanodeDescriptor datanode1 = NameNodeRaidTestUtil.getDatanodeMap(namesystem).values().iterator().next(); String source = "/dir/file"; String parity = xorPrefix + source; final Path parityPath = new Path(parity); DFSTestUtil.createFile(fs, parityPath, 3, (short) 1, 0L); DFSTestUtil.waitReplication(fs, parityPath, (short) 1); // start one more datanode cluster.startDataNodes(conf, 1, true, null, rack2, host2, null); DatanodeDescriptor datanode2 = null; for (DatanodeDescriptor d : NameNodeRaidTestUtil.getDatanodeMap(namesystem).values()) { if (!d.getName().equals(datanode1.getName())) { datanode2 = d; } } Assert.assertTrue(datanode2 != null); cluster.waitActive(); final Path sourcePath = new Path(source); DFSTestUtil.createFile(fs, sourcePath, 5, (short) 2, 0L); DFSTestUtil.waitReplication(fs, sourcePath, (short) 2); refreshPolicy(); Assert.assertEquals(parity, policy.getParityFile(source)); Assert.assertEquals(source, policy.getSourceFile(parity, xorPrefix)); List sourceBlocks = getBlocks(namesystem, source); List parityBlocks = getBlocks(namesystem, parity); Assert.assertEquals(5, sourceBlocks.size()); Assert.assertEquals(3, parityBlocks.size()); // verify the result of getCompanionBlocks() Collection companionBlocks; companionBlocks = getCompanionBlocks(namesystem, policy, sourceBlocks.get(0).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 0, 1 }, new int[] { 0 }); companionBlocks = getCompanionBlocks(namesystem, policy, sourceBlocks.get(1).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 0, 1 }, new int[] { 0 }); companionBlocks = getCompanionBlocks(namesystem, policy, sourceBlocks.get(2).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 2, 3 }, new int[] { 1 }); companionBlocks = getCompanionBlocks(namesystem, policy, sourceBlocks.get(3).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 2, 3 }, new int[] { 1 }); companionBlocks = getCompanionBlocks(namesystem, policy, sourceBlocks.get(4).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 4 }, new int[] { 2 }); companionBlocks = getCompanionBlocks(namesystem, policy, parityBlocks.get(0).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 0, 1 }, new int[] { 0 }); companionBlocks = getCompanionBlocks(namesystem, policy, parityBlocks.get(1).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 2, 3 }, new int[] { 1 }); companionBlocks = getCompanionBlocks(namesystem, policy, parityBlocks.get(2).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 4 }, new int[] { 2 }); // Set the policy back to raid policy. We have to create a new object // here to clear the block location cache refreshPolicy(); setBlockPlacementPolicy(namesystem, policy); // verify policy deletes the correct blocks. companion blocks should be // evenly distributed. fs.setReplication(sourcePath, (short) 1); DFSTestUtil.waitReplication(fs, sourcePath, (short) 1); Map counters = new HashMap(); refreshPolicy(); for (int i = 0; i < parityBlocks.size(); i++) { companionBlocks = getCompanionBlocks(namesystem, policy, parityBlocks.get(i).getBlock()); counters = BlockPlacementPolicyRaid.countCompanionBlocks(companionBlocks, false); Assert.assertTrue(counters.get(datanode1.getName()) >= 1 && counters.get(datanode1.getName()) <= 2); Assert.assertTrue(counters.get(datanode1.getName()) + counters.get(datanode2.getName()) == companionBlocks.size()); counters = BlockPlacementPolicyRaid.countCompanionBlocks(companionBlocks, true); Assert.assertTrue(counters.get(datanode1.getParent().getName()) >= 1 && counters.get(datanode1.getParent().getName()) <= 2); Assert.assertTrue(counters.get(datanode1.getParent().getName()) + counters.get(datanode2.getParent().getName()) == companionBlocks.size()); } } finally { if (cluster != null) { cluster.shutdown(); } }
5705 -1009651940apache/hadoopTsz-wo Szecabec474e0f4c525ba80bd7c8dede1a8d76e4f39 // verify policy deletes the correct blocks. companion blocks should be // evenly distributed. start one more datanode SATD_REMOVED testDeleteReplica() public void testDeleteReplica() throws IOException setupCluster(); try { // Set the policy to default policy to place the block in the default way setBlockPlacementPolicy(namesystem, new BlockPlacementPolicyDefault(conf, namesystem, namesystem.clusterMap)); DatanodeDescriptor datanode1 = NameNodeRaidTestUtil.getDatanodeMap(namesystem).values().iterator().next(); String source = "/dir/file"; String parity = xorPrefix + source; final Path parityPath = new Path(parity); DFSTestUtil.createFile(fs, parityPath, 3, (short) 1, 0L); DFSTestUtil.waitReplication(fs, parityPath, (short) 1); // start one more datanode cluster.startDataNodes(conf, 1, true, null, rack2, host2, null); DatanodeDescriptor datanode2 = null; for (DatanodeDescriptor d : NameNodeRaidTestUtil.getDatanodeMap(namesystem).values()) { if (!d.getName().equals(datanode1.getName())) { datanode2 = d; } } Assert.assertTrue(datanode2 != null); cluster.waitActive(); final Path sourcePath = new Path(source); DFSTestUtil.createFile(fs, sourcePath, 5, (short) 2, 0L); DFSTestUtil.waitReplication(fs, sourcePath, (short) 2); refreshPolicy(); Assert.assertEquals(parity, policy.getParityFile(source)); Assert.assertEquals(source, policy.getSourceFile(parity, xorPrefix)); List sourceBlocks = getBlocks(namesystem, source); List parityBlocks = getBlocks(namesystem, parity); Assert.assertEquals(5, sourceBlocks.size()); Assert.assertEquals(3, parityBlocks.size()); // verify the result of getCompanionBlocks() Collection companionBlocks; companionBlocks = getCompanionBlocks(namesystem, policy, sourceBlocks.get(0).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 0, 1 }, new int[] { 0 }); companionBlocks = getCompanionBlocks(namesystem, policy, sourceBlocks.get(1).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 0, 1 }, new int[] { 0 }); companionBlocks = getCompanionBlocks(namesystem, policy, sourceBlocks.get(2).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 2, 3 }, new int[] { 1 }); companionBlocks = getCompanionBlocks(namesystem, policy, sourceBlocks.get(3).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 2, 3 }, new int[] { 1 }); companionBlocks = getCompanionBlocks(namesystem, policy, sourceBlocks.get(4).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 4 }, new int[] { 2 }); companionBlocks = getCompanionBlocks(namesystem, policy, parityBlocks.get(0).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 0, 1 }, new int[] { 0 }); companionBlocks = getCompanionBlocks(namesystem, policy, parityBlocks.get(1).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 2, 3 }, new int[] { 1 }); companionBlocks = getCompanionBlocks(namesystem, policy, parityBlocks.get(2).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 4 }, new int[] { 2 }); // Set the policy back to raid policy. We have to create a new object // here to clear the block location cache refreshPolicy(); setBlockPlacementPolicy(namesystem, policy); // verify policy deletes the correct blocks. companion blocks should be // evenly distributed. fs.setReplication(sourcePath, (short) 1); DFSTestUtil.waitReplication(fs, sourcePath, (short) 1); Map counters = new HashMap(); refreshPolicy(); for (int i = 0; i < parityBlocks.size(); i++) { companionBlocks = getCompanionBlocks(namesystem, policy, parityBlocks.get(i).getBlock()); counters = BlockPlacementPolicyRaid.countCompanionBlocks(companionBlocks, false); Assert.assertTrue(counters.get(datanode1.getName()) >= 1 && counters.get(datanode1.getName()) <= 2); Assert.assertTrue(counters.get(datanode1.getName()) + counters.get(datanode2.getName()) == companionBlocks.size()); counters = BlockPlacementPolicyRaid.countCompanionBlocks(companionBlocks, true); Assert.assertTrue(counters.get(datanode1.getParent().getName()) >= 1 && counters.get(datanode1.getParent().getName()) <= 2); Assert.assertTrue(counters.get(datanode1.getParent().getName()) + counters.get(datanode2.getParent().getName()) == companionBlocks.size()); } } finally { if (cluster != null) { cluster.shutdown(); } }
5703 -1009651941apache/hadoopTsz-wo Szecabec474e0f4c525ba80bd7c8dede1a8d76e4f39 // verify policy deletes the correct blocks. companion blocks should be // evenly distributed. Set the policy to default policy to place the block in the default way SATD_CHANGED testDeleteReplica() public void testDeleteReplica() throws IOException setupCluster(); try { // Set the policy to default policy to place the block in the default way setBlockPlacementPolicy(namesystem, new BlockPlacementPolicyDefault(conf, namesystem, namesystem.clusterMap)); DatanodeDescriptor datanode1 = NameNodeRaidTestUtil.getDatanodeMap(namesystem).values().iterator().next(); String source = "/dir/file"; String parity = xorPrefix + source; final Path parityPath = new Path(parity); DFSTestUtil.createFile(fs, parityPath, 3, (short) 1, 0L); DFSTestUtil.waitReplication(fs, parityPath, (short) 1); // start one more datanode cluster.startDataNodes(conf, 1, true, null, rack2, host2, null); DatanodeDescriptor datanode2 = null; for (DatanodeDescriptor d : NameNodeRaidTestUtil.getDatanodeMap(namesystem).values()) { if (!d.getName().equals(datanode1.getName())) { datanode2 = d; } } Assert.assertTrue(datanode2 != null); cluster.waitActive(); final Path sourcePath = new Path(source); DFSTestUtil.createFile(fs, sourcePath, 5, (short) 2, 0L); DFSTestUtil.waitReplication(fs, sourcePath, (short) 2); refreshPolicy(); Assert.assertEquals(parity, policy.getParityFile(source)); Assert.assertEquals(source, policy.getSourceFile(parity, xorPrefix)); List sourceBlocks = getBlocks(namesystem, source); List parityBlocks = getBlocks(namesystem, parity); Assert.assertEquals(5, sourceBlocks.size()); Assert.assertEquals(3, parityBlocks.size()); // verify the result of getCompanionBlocks() Collection companionBlocks; companionBlocks = getCompanionBlocks(namesystem, policy, sourceBlocks.get(0).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 0, 1 }, new int[] { 0 }); companionBlocks = getCompanionBlocks(namesystem, policy, sourceBlocks.get(1).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 0, 1 }, new int[] { 0 }); companionBlocks = getCompanionBlocks(namesystem, policy, sourceBlocks.get(2).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 2, 3 }, new int[] { 1 }); companionBlocks = getCompanionBlocks(namesystem, policy, sourceBlocks.get(3).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 2, 3 }, new int[] { 1 }); companionBlocks = getCompanionBlocks(namesystem, policy, sourceBlocks.get(4).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 4 }, new int[] { 2 }); companionBlocks = getCompanionBlocks(namesystem, policy, parityBlocks.get(0).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 0, 1 }, new int[] { 0 }); companionBlocks = getCompanionBlocks(namesystem, policy, parityBlocks.get(1).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 2, 3 }, new int[] { 1 }); companionBlocks = getCompanionBlocks(namesystem, policy, parityBlocks.get(2).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 4 }, new int[] { 2 }); // Set the policy back to raid policy. We have to create a new object // here to clear the block location cache refreshPolicy(); setBlockPlacementPolicy(namesystem, policy); // verify policy deletes the correct blocks. companion blocks should be // evenly distributed. fs.setReplication(sourcePath, (short) 1); DFSTestUtil.waitReplication(fs, sourcePath, (short) 1); Map counters = new HashMap(); refreshPolicy(); for (int i = 0; i < parityBlocks.size(); i++) { companionBlocks = getCompanionBlocks(namesystem, policy, parityBlocks.get(i).getBlock()); counters = BlockPlacementPolicyRaid.countCompanionBlocks(companionBlocks, false); Assert.assertTrue(counters.get(datanode1.getName()) >= 1 && counters.get(datanode1.getName()) <= 2); Assert.assertTrue(counters.get(datanode1.getName()) + counters.get(datanode2.getName()) == companionBlocks.size()); counters = BlockPlacementPolicyRaid.countCompanionBlocks(companionBlocks, true); Assert.assertTrue(counters.get(datanode1.getParent().getName()) >= 1 && counters.get(datanode1.getParent().getName()) <= 2); Assert.assertTrue(counters.get(datanode1.getParent().getName()) + counters.get(datanode2.getParent().getName()) == companionBlocks.size()); } } finally { if (cluster != null) { cluster.shutdown(); } }
5701 -1009651942apache/hadoopTsz-wo Szecabec474e0f4c525ba80bd7c8dede1a8d76e4f39 // verify policy deletes the correct blocks. companion blocks should be // evenly distributed. Set the policy to default policy to place the block in the default way SATD_CHANGED testGetCompanionBLocks() public void testGetCompanionBLocks() throws IOException setupCluster(); try { String file1 = "/dir/file1"; String file2 = "/raid/dir/file2"; String file3 = "/raidrs/dir/file3"; // Set the policy to default policy to place the block in the default way setBlockPlacementPolicy(namesystem, new BlockPlacementPolicyDefault(conf, namesystem, namesystem.clusterMap)); DFSTestUtil.createFile(fs, new Path(file1), 3, (short) 1, 0L); DFSTestUtil.createFile(fs, new Path(file2), 4, (short) 1, 0L); DFSTestUtil.createFile(fs, new Path(file3), 8, (short) 1, 0L); Collection companionBlocks; companionBlocks = getCompanionBlocks(namesystem, policy, getBlocks(namesystem, file1).get(0).getBlock()); Assert.assertTrue(companionBlocks == null || companionBlocks.size() == 0); companionBlocks = getCompanionBlocks(namesystem, policy, getBlocks(namesystem, file1).get(2).getBlock()); Assert.assertTrue(companionBlocks == null || companionBlocks.size() == 0); companionBlocks = getCompanionBlocks(namesystem, policy, getBlocks(namesystem, file2).get(0).getBlock()); Assert.assertEquals(1, companionBlocks.size()); companionBlocks = getCompanionBlocks(namesystem, policy, getBlocks(namesystem, file2).get(3).getBlock()); Assert.assertEquals(1, companionBlocks.size()); int rsParityLength = RaidNode.rsParityLength(conf); companionBlocks = getCompanionBlocks(namesystem, policy, getBlocks(namesystem, file3).get(0).getBlock()); Assert.assertEquals(rsParityLength, companionBlocks.size()); companionBlocks = getCompanionBlocks(namesystem, policy, getBlocks(namesystem, file3).get(4).getBlock()); Assert.assertEquals(rsParityLength, companionBlocks.size()); companionBlocks = getCompanionBlocks(namesystem, policy, getBlocks(namesystem, file3).get(6).getBlock()); Assert.assertEquals(2, companionBlocks.size()); } finally { if (cluster != null) { cluster.shutdown(); } }
5699 -1009651943apache/hadoopTsz-wo Szecabec474e0f4c525ba80bd7c8dede1a8d76e4f39 // verify policy deletes the correct blocks. companion blocks should be // evenly distributed. test full path cache SATD_REMOVED testCachedPathNames() public void testCachedPathNames() throws IOException setupCluster(); try { String file1 = "/dir/file1"; String file2 = "/dir/file2"; DFSTestUtil.createFile(fs, new Path(file1), 3, (short) 1, 0L); DFSTestUtil.createFile(fs, new Path(file2), 4, (short) 1, 0L); // test full path cache CachedFullPathNames cachedFullPathNames = new CachedFullPathNames(namesystem); FSInodeInfo inode1 = null; FSInodeInfo inode2 = null; NameNodeRaidTestUtil.readLock(namesystem.dir); try { inode1 = NameNodeRaidTestUtil.getNode(namesystem.dir, file1, true); inode2 = NameNodeRaidTestUtil.getNode(namesystem.dir, file2, true); } finally { NameNodeRaidTestUtil.readUnLock(namesystem.dir); } verifyCachedFullPathNameResult(cachedFullPathNames, inode1); verifyCachedFullPathNameResult(cachedFullPathNames, inode1); verifyCachedFullPathNameResult(cachedFullPathNames, inode2); verifyCachedFullPathNameResult(cachedFullPathNames, inode2); try { Thread.sleep(1200L); } catch (InterruptedException e) { } verifyCachedFullPathNameResult(cachedFullPathNames, inode2); verifyCachedFullPathNameResult(cachedFullPathNames, inode1); } finally { if (cluster != null) { cluster.shutdown(); } }
5698 -1009651944apache/hadoopTsz-wo Szecabec474e0f4c525ba80bd7c8dede1a8d76e4f39 // verify policy deletes the correct blocks. companion blocks should be // evenly distributed. test blocks cache SATD_REMOVED testCachedBlocks() public void testCachedBlocks() throws IOException setupCluster(); try { String file1 = "/dir/file1"; String file2 = "/dir/file2"; DFSTestUtil.createFile(fs, new Path(file1), 3, (short) 1, 0L); DFSTestUtil.createFile(fs, new Path(file2), 4, (short) 1, 0L); // test blocks cache CachedLocatedBlocks cachedBlocks = new CachedLocatedBlocks(namesystem); verifyCachedBlocksResult(cachedBlocks, namesystem, file1); verifyCachedBlocksResult(cachedBlocks, namesystem, file1); verifyCachedBlocksResult(cachedBlocks, namesystem, file2); verifyCachedBlocksResult(cachedBlocks, namesystem, file2); try { Thread.sleep(1200L); } catch (InterruptedException e) { } verifyCachedBlocksResult(cachedBlocks, namesystem, file2); verifyCachedBlocksResult(cachedBlocks, namesystem, file1); } finally { if (cluster != null) { cluster.shutdown(); } }
5696 -1009651945apache/hadoopTsz-wo Szecabec474e0f4c525ba80bd7c8dede1a8d76e4f39 // verify policy deletes the correct blocks. companion blocks should be // evenly distributed. verify that every adjacent 4 blocks are on differnt nodes SATD_REMOVED testChooseTargetForHarRaidFile() public void testChooseTargetForHarRaidFile() throws IOException setupCluster(); try { String[] racks = { "/rack2", "/rack2", "/rack2", "/rack2", "/rack2", "/rack2" }; String[] hosts = { "host2.rack2.com", "host3.rack2.com", "host4.rack2.com", "host5.rack2.com", "host6.rack2.com", "host7.rack2.com" }; cluster.startDataNodes(conf, 6, true, null, racks, hosts, null); String harParity = raidrsHarTempPrefix + "/dir/file"; int numBlocks = 11; DFSTestUtil.createFile(fs, new Path(harParity), numBlocks, (short) 1, 0L); DFSTestUtil.waitReplication(fs, new Path(harParity), (short) 1); FileStatus stat = fs.getFileStatus(new Path(harParity)); BlockLocation[] loc = fs.getFileBlockLocations(stat, 0, stat.getLen()); int rsParityLength = RaidNode.rsParityLength(conf); for (int i = 0; i < numBlocks - rsParityLength; i++) { Set locations = new HashSet(); for (int j = 0; j < rsParityLength; j++) { for (int k = 0; k < loc[i + j].getNames().length; k++) { // verify that every adjacent 4 blocks are on differnt nodes String name = loc[i + j].getNames()[k]; LOG.info("Har Raid block location: " + name); Assert.assertTrue(locations.add(name)); } } } } finally { if (cluster != null) { cluster.shutdown(); } }
5694 -1009651946apache/hadoopTsz-wo Szecabec474e0f4c525ba80bd7c8dede1a8d76e4f39 // verify policy deletes the correct blocks. companion blocks should be // evenly distributed. start 3 more datanodes SATD_REMOVED testChooseTargetForRaidFile() public void testChooseTargetForRaidFile() throws IOException setupCluster(); try { String src = "/dir/file"; String parity = raidrsTempPrefix + src; DFSTestUtil.createFile(fs, new Path(src), 4, (short) 1, 0L); DFSTestUtil.waitReplication(fs, new Path(src), (short) 1); refreshPolicy(); setBlockPlacementPolicy(namesystem, policy); // start 3 more datanodes String[] racks = { "/rack2", "/rack2", "/rack2", "/rack2", "/rack2", "/rack2" }; String[] hosts = { "host2.rack2.com", "host3.rack2.com", "host4.rack2.com", "host5.rack2.com", "host6.rack2.com", "host7.rack2.com" }; cluster.startDataNodes(conf, 6, true, null, racks, hosts, null); int numBlocks = 6; DFSTestUtil.createFile(fs, new Path(parity), numBlocks, (short) 2, 0L); DFSTestUtil.waitReplication(fs, new Path(parity), (short) 2); FileStatus srcStat = fs.getFileStatus(new Path(src)); BlockLocation[] srcLoc = fs.getFileBlockLocations(srcStat, 0, srcStat.getLen()); FileStatus parityStat = fs.getFileStatus(new Path(parity)); BlockLocation[] parityLoc = fs.getFileBlockLocations(parityStat, 0, parityStat.getLen()); int parityLen = RaidNode.rsParityLength(conf); for (int i = 0; i < numBlocks / parityLen; i++) { Set locations = new HashSet(); for (int j = 0; j < srcLoc.length; j++) { String[] names = srcLoc[j].getNames(); for (int k = 0; k < names.length; k++) { LOG.info("Source block location: " + names[k]); locations.add(names[k]); } } for (int j = 0; j < parityLen; j++) { String[] names = parityLoc[j + i * parityLen].getNames(); for (int k = 0; k < names.length; k++) { LOG.info("Parity block location: " + names[k]); Assert.assertTrue(locations.add(names[k])); } } } } finally { if (cluster != null) { cluster.shutdown(); } }
5692 -1009651975apache/hadoopTsz-wo Szecabec474e0f4c525ba80bd7c8dede1a8d76e4f39 // verify policy deletes the correct blocks. companion blocks should be // evenly distributed. start the cluster with one datanode first SATD_REMOVED setupCluster() protected void setupCluster() throws IOException conf = new Configuration(); conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 1000L); conf.set("dfs.replication.pending.timeout.sec", "2"); conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 1L); conf.set("dfs.block.replicator.classname", "org.apache.hadoop.hdfs.server.namenode.BlockPlacementPolicyRaid"); conf.set(RaidNode.STRIPE_LENGTH_KEY, "2"); conf.set(RaidNode.RS_PARITY_LENGTH_KEY, "3"); conf.setInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, 1); // start the cluster with one datanode first cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).format(true).racks(rack1).hosts(host1).build(); cluster.waitActive(); namesystem = cluster.getNameNode().getNamesystem(); Assert.assertTrue("BlockPlacementPolicy type is not correct.", namesystem.blockManager.replicator instanceof BlockPlacementPolicyRaid); policy = (BlockPlacementPolicyRaid) namesystem.blockManager.replicator; fs = cluster.getFileSystem(); xorPrefix = RaidNode.xorDestinationPath(conf).toUri().getPath(); raidTempPrefix = RaidNode.xorTempPrefix(conf); raidrsTempPrefix = RaidNode.rsTempPrefix(conf); raidrsHarTempPrefix = RaidNode.rsHarTempPrefix(conf);
5691 -1009651947apache/hadoopTsz-wo Szecabec474e0f4c525ba80bd7c8dede1a8d76e4f39 // Set the policy to default policy to place the block in the default way setBlockPlacementPolicy(namesystem, new BlockPlacementPolicyDefault( create a new BlockPlacementPolicyRaid to clear the cache SATD_REMOVED refreshPolicy() private void refreshPolicy() policy = new BlockPlacementPolicyRaid(); policy.initialize(conf, namesystem, namesystem.clusterMap);
5688 -1009651948apache/hadoopTsz-wo Szecabec474e0f4c525ba80bd7c8dede1a8d76e4f39 // Set the policy to default policy to place the block in the default way setBlockPlacementPolicy(namesystem, new BlockPlacementPolicyDefault( verify policy deletes the correct blocks. companion blocks should be evenly distributed. SATD_CHANGED testDeleteReplica() public void testDeleteReplica() throws IOException setupCluster(); try { // Set the policy to default policy to place the block in the default way setBlockPlacementPolicy(namesystem, new BlockPlacementPolicyDefault(conf, namesystem, namesystem.clusterMap)); DatanodeDescriptor datanode1 = NameNodeRaidTestUtil.getDatanodeMap(namesystem).values().iterator().next(); String source = "/dir/file"; String parity = xorPrefix + source; final Path parityPath = new Path(parity); DFSTestUtil.createFile(fs, parityPath, 3, (short) 1, 0L); DFSTestUtil.waitReplication(fs, parityPath, (short) 1); // start one more datanode cluster.startDataNodes(conf, 1, true, null, rack2, host2, null); DatanodeDescriptor datanode2 = null; for (DatanodeDescriptor d : NameNodeRaidTestUtil.getDatanodeMap(namesystem).values()) { if (!d.getName().equals(datanode1.getName())) { datanode2 = d; } } Assert.assertTrue(datanode2 != null); cluster.waitActive(); final Path sourcePath = new Path(source); DFSTestUtil.createFile(fs, sourcePath, 5, (short) 2, 0L); DFSTestUtil.waitReplication(fs, sourcePath, (short) 2); refreshPolicy(); Assert.assertEquals(parity, policy.getParityFile(source)); Assert.assertEquals(source, policy.getSourceFile(parity, xorPrefix)); List sourceBlocks = getBlocks(namesystem, source); List parityBlocks = getBlocks(namesystem, parity); Assert.assertEquals(5, sourceBlocks.size()); Assert.assertEquals(3, parityBlocks.size()); // verify the result of getCompanionBlocks() Collection companionBlocks; companionBlocks = getCompanionBlocks(namesystem, policy, sourceBlocks.get(0).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 0, 1 }, new int[] { 0 }); companionBlocks = getCompanionBlocks(namesystem, policy, sourceBlocks.get(1).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 0, 1 }, new int[] { 0 }); companionBlocks = getCompanionBlocks(namesystem, policy, sourceBlocks.get(2).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 2, 3 }, new int[] { 1 }); companionBlocks = getCompanionBlocks(namesystem, policy, sourceBlocks.get(3).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 2, 3 }, new int[] { 1 }); companionBlocks = getCompanionBlocks(namesystem, policy, sourceBlocks.get(4).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 4 }, new int[] { 2 }); companionBlocks = getCompanionBlocks(namesystem, policy, parityBlocks.get(0).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 0, 1 }, new int[] { 0 }); companionBlocks = getCompanionBlocks(namesystem, policy, parityBlocks.get(1).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 2, 3 }, new int[] { 1 }); companionBlocks = getCompanionBlocks(namesystem, policy, parityBlocks.get(2).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 4 }, new int[] { 2 }); // Set the policy back to raid policy. We have to create a new object // here to clear the block location cache refreshPolicy(); setBlockPlacementPolicy(namesystem, policy); // verify policy deletes the correct blocks. companion blocks should be // evenly distributed. fs.setReplication(sourcePath, (short) 1); DFSTestUtil.waitReplication(fs, sourcePath, (short) 1); Map counters = new HashMap(); refreshPolicy(); for (int i = 0; i < parityBlocks.size(); i++) { companionBlocks = getCompanionBlocks(namesystem, policy, parityBlocks.get(i).getBlock()); counters = BlockPlacementPolicyRaid.countCompanionBlocks(companionBlocks, false); Assert.assertTrue(counters.get(datanode1.getName()) >= 1 && counters.get(datanode1.getName()) <= 2); Assert.assertTrue(counters.get(datanode1.getName()) + counters.get(datanode2.getName()) == companionBlocks.size()); counters = BlockPlacementPolicyRaid.countCompanionBlocks(companionBlocks, true); Assert.assertTrue(counters.get(datanode1.getParent().getName()) >= 1 && counters.get(datanode1.getParent().getName()) <= 2); Assert.assertTrue(counters.get(datanode1.getParent().getName()) + counters.get(datanode2.getParent().getName()) == companionBlocks.size()); } } finally { if (cluster != null) { cluster.shutdown(); } }
5687 -1009651949apache/hadoopTsz-wo Szecabec474e0f4c525ba80bd7c8dede1a8d76e4f39 // Set the policy to default policy to place the block in the default way setBlockPlacementPolicy(namesystem, new BlockPlacementPolicyDefault( Set the policy back to raid policy. We have to create a new object here to clear the block location cache SATD_REMOVED testDeleteReplica() public void testDeleteReplica() throws IOException setupCluster(); try { // Set the policy to default policy to place the block in the default way setBlockPlacementPolicy(namesystem, new BlockPlacementPolicyDefault(conf, namesystem, namesystem.clusterMap)); DatanodeDescriptor datanode1 = NameNodeRaidTestUtil.getDatanodeMap(namesystem).values().iterator().next(); String source = "/dir/file"; String parity = xorPrefix + source; final Path parityPath = new Path(parity); DFSTestUtil.createFile(fs, parityPath, 3, (short) 1, 0L); DFSTestUtil.waitReplication(fs, parityPath, (short) 1); // start one more datanode cluster.startDataNodes(conf, 1, true, null, rack2, host2, null); DatanodeDescriptor datanode2 = null; for (DatanodeDescriptor d : NameNodeRaidTestUtil.getDatanodeMap(namesystem).values()) { if (!d.getName().equals(datanode1.getName())) { datanode2 = d; } } Assert.assertTrue(datanode2 != null); cluster.waitActive(); final Path sourcePath = new Path(source); DFSTestUtil.createFile(fs, sourcePath, 5, (short) 2, 0L); DFSTestUtil.waitReplication(fs, sourcePath, (short) 2); refreshPolicy(); Assert.assertEquals(parity, policy.getParityFile(source)); Assert.assertEquals(source, policy.getSourceFile(parity, xorPrefix)); List sourceBlocks = getBlocks(namesystem, source); List parityBlocks = getBlocks(namesystem, parity); Assert.assertEquals(5, sourceBlocks.size()); Assert.assertEquals(3, parityBlocks.size()); // verify the result of getCompanionBlocks() Collection companionBlocks; companionBlocks = getCompanionBlocks(namesystem, policy, sourceBlocks.get(0).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 0, 1 }, new int[] { 0 }); companionBlocks = getCompanionBlocks(namesystem, policy, sourceBlocks.get(1).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 0, 1 }, new int[] { 0 }); companionBlocks = getCompanionBlocks(namesystem, policy, sourceBlocks.get(2).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 2, 3 }, new int[] { 1 }); companionBlocks = getCompanionBlocks(namesystem, policy, sourceBlocks.get(3).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 2, 3 }, new int[] { 1 }); companionBlocks = getCompanionBlocks(namesystem, policy, sourceBlocks.get(4).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 4 }, new int[] { 2 }); companionBlocks = getCompanionBlocks(namesystem, policy, parityBlocks.get(0).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 0, 1 }, new int[] { 0 }); companionBlocks = getCompanionBlocks(namesystem, policy, parityBlocks.get(1).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 2, 3 }, new int[] { 1 }); companionBlocks = getCompanionBlocks(namesystem, policy, parityBlocks.get(2).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 4 }, new int[] { 2 }); // Set the policy back to raid policy. We have to create a new object // here to clear the block location cache refreshPolicy(); setBlockPlacementPolicy(namesystem, policy); // verify policy deletes the correct blocks. companion blocks should be // evenly distributed. fs.setReplication(sourcePath, (short) 1); DFSTestUtil.waitReplication(fs, sourcePath, (short) 1); Map counters = new HashMap(); refreshPolicy(); for (int i = 0; i < parityBlocks.size(); i++) { companionBlocks = getCompanionBlocks(namesystem, policy, parityBlocks.get(i).getBlock()); counters = BlockPlacementPolicyRaid.countCompanionBlocks(companionBlocks, false); Assert.assertTrue(counters.get(datanode1.getName()) >= 1 && counters.get(datanode1.getName()) <= 2); Assert.assertTrue(counters.get(datanode1.getName()) + counters.get(datanode2.getName()) == companionBlocks.size()); counters = BlockPlacementPolicyRaid.countCompanionBlocks(companionBlocks, true); Assert.assertTrue(counters.get(datanode1.getParent().getName()) >= 1 && counters.get(datanode1.getParent().getName()) <= 2); Assert.assertTrue(counters.get(datanode1.getParent().getName()) + counters.get(datanode2.getParent().getName()) == companionBlocks.size()); } } finally { if (cluster != null) { cluster.shutdown(); } }
5685 -1009651950apache/hadoopTsz-wo Szecabec474e0f4c525ba80bd7c8dede1a8d76e4f39 // Set the policy to default policy to place the block in the default way setBlockPlacementPolicy(namesystem, new BlockPlacementPolicyDefault( verify the result of getCompanionBlocks() SATD_REMOVED testDeleteReplica() public void testDeleteReplica() throws IOException setupCluster(); try { // Set the policy to default policy to place the block in the default way setBlockPlacementPolicy(namesystem, new BlockPlacementPolicyDefault(conf, namesystem, namesystem.clusterMap)); DatanodeDescriptor datanode1 = NameNodeRaidTestUtil.getDatanodeMap(namesystem).values().iterator().next(); String source = "/dir/file"; String parity = xorPrefix + source; final Path parityPath = new Path(parity); DFSTestUtil.createFile(fs, parityPath, 3, (short) 1, 0L); DFSTestUtil.waitReplication(fs, parityPath, (short) 1); // start one more datanode cluster.startDataNodes(conf, 1, true, null, rack2, host2, null); DatanodeDescriptor datanode2 = null; for (DatanodeDescriptor d : NameNodeRaidTestUtil.getDatanodeMap(namesystem).values()) { if (!d.getName().equals(datanode1.getName())) { datanode2 = d; } } Assert.assertTrue(datanode2 != null); cluster.waitActive(); final Path sourcePath = new Path(source); DFSTestUtil.createFile(fs, sourcePath, 5, (short) 2, 0L); DFSTestUtil.waitReplication(fs, sourcePath, (short) 2); refreshPolicy(); Assert.assertEquals(parity, policy.getParityFile(source)); Assert.assertEquals(source, policy.getSourceFile(parity, xorPrefix)); List sourceBlocks = getBlocks(namesystem, source); List parityBlocks = getBlocks(namesystem, parity); Assert.assertEquals(5, sourceBlocks.size()); Assert.assertEquals(3, parityBlocks.size()); // verify the result of getCompanionBlocks() Collection companionBlocks; companionBlocks = getCompanionBlocks(namesystem, policy, sourceBlocks.get(0).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 0, 1 }, new int[] { 0 }); companionBlocks = getCompanionBlocks(namesystem, policy, sourceBlocks.get(1).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 0, 1 }, new int[] { 0 }); companionBlocks = getCompanionBlocks(namesystem, policy, sourceBlocks.get(2).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 2, 3 }, new int[] { 1 }); companionBlocks = getCompanionBlocks(namesystem, policy, sourceBlocks.get(3).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 2, 3 }, new int[] { 1 }); companionBlocks = getCompanionBlocks(namesystem, policy, sourceBlocks.get(4).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 4 }, new int[] { 2 }); companionBlocks = getCompanionBlocks(namesystem, policy, parityBlocks.get(0).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 0, 1 }, new int[] { 0 }); companionBlocks = getCompanionBlocks(namesystem, policy, parityBlocks.get(1).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 2, 3 }, new int[] { 1 }); companionBlocks = getCompanionBlocks(namesystem, policy, parityBlocks.get(2).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 4 }, new int[] { 2 }); // Set the policy back to raid policy. We have to create a new object // here to clear the block location cache refreshPolicy(); setBlockPlacementPolicy(namesystem, policy); // verify policy deletes the correct blocks. companion blocks should be // evenly distributed. fs.setReplication(sourcePath, (short) 1); DFSTestUtil.waitReplication(fs, sourcePath, (short) 1); Map counters = new HashMap(); refreshPolicy(); for (int i = 0; i < parityBlocks.size(); i++) { companionBlocks = getCompanionBlocks(namesystem, policy, parityBlocks.get(i).getBlock()); counters = BlockPlacementPolicyRaid.countCompanionBlocks(companionBlocks, false); Assert.assertTrue(counters.get(datanode1.getName()) >= 1 && counters.get(datanode1.getName()) <= 2); Assert.assertTrue(counters.get(datanode1.getName()) + counters.get(datanode2.getName()) == companionBlocks.size()); counters = BlockPlacementPolicyRaid.countCompanionBlocks(companionBlocks, true); Assert.assertTrue(counters.get(datanode1.getParent().getName()) >= 1 && counters.get(datanode1.getParent().getName()) <= 2); Assert.assertTrue(counters.get(datanode1.getParent().getName()) + counters.get(datanode2.getParent().getName()) == companionBlocks.size()); } } finally { if (cluster != null) { cluster.shutdown(); } }
5683 -1009651951apache/hadoopTsz-wo Szecabec474e0f4c525ba80bd7c8dede1a8d76e4f39 // Set the policy to default policy to place the block in the default way setBlockPlacementPolicy(namesystem, new BlockPlacementPolicyDefault( start one more datanode SATD_REMOVED testDeleteReplica() public void testDeleteReplica() throws IOException setupCluster(); try { // Set the policy to default policy to place the block in the default way setBlockPlacementPolicy(namesystem, new BlockPlacementPolicyDefault(conf, namesystem, namesystem.clusterMap)); DatanodeDescriptor datanode1 = NameNodeRaidTestUtil.getDatanodeMap(namesystem).values().iterator().next(); String source = "/dir/file"; String parity = xorPrefix + source; final Path parityPath = new Path(parity); DFSTestUtil.createFile(fs, parityPath, 3, (short) 1, 0L); DFSTestUtil.waitReplication(fs, parityPath, (short) 1); // start one more datanode cluster.startDataNodes(conf, 1, true, null, rack2, host2, null); DatanodeDescriptor datanode2 = null; for (DatanodeDescriptor d : NameNodeRaidTestUtil.getDatanodeMap(namesystem).values()) { if (!d.getName().equals(datanode1.getName())) { datanode2 = d; } } Assert.assertTrue(datanode2 != null); cluster.waitActive(); final Path sourcePath = new Path(source); DFSTestUtil.createFile(fs, sourcePath, 5, (short) 2, 0L); DFSTestUtil.waitReplication(fs, sourcePath, (short) 2); refreshPolicy(); Assert.assertEquals(parity, policy.getParityFile(source)); Assert.assertEquals(source, policy.getSourceFile(parity, xorPrefix)); List sourceBlocks = getBlocks(namesystem, source); List parityBlocks = getBlocks(namesystem, parity); Assert.assertEquals(5, sourceBlocks.size()); Assert.assertEquals(3, parityBlocks.size()); // verify the result of getCompanionBlocks() Collection companionBlocks; companionBlocks = getCompanionBlocks(namesystem, policy, sourceBlocks.get(0).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 0, 1 }, new int[] { 0 }); companionBlocks = getCompanionBlocks(namesystem, policy, sourceBlocks.get(1).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 0, 1 }, new int[] { 0 }); companionBlocks = getCompanionBlocks(namesystem, policy, sourceBlocks.get(2).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 2, 3 }, new int[] { 1 }); companionBlocks = getCompanionBlocks(namesystem, policy, sourceBlocks.get(3).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 2, 3 }, new int[] { 1 }); companionBlocks = getCompanionBlocks(namesystem, policy, sourceBlocks.get(4).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 4 }, new int[] { 2 }); companionBlocks = getCompanionBlocks(namesystem, policy, parityBlocks.get(0).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 0, 1 }, new int[] { 0 }); companionBlocks = getCompanionBlocks(namesystem, policy, parityBlocks.get(1).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 2, 3 }, new int[] { 1 }); companionBlocks = getCompanionBlocks(namesystem, policy, parityBlocks.get(2).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 4 }, new int[] { 2 }); // Set the policy back to raid policy. We have to create a new object // here to clear the block location cache refreshPolicy(); setBlockPlacementPolicy(namesystem, policy); // verify policy deletes the correct blocks. companion blocks should be // evenly distributed. fs.setReplication(sourcePath, (short) 1); DFSTestUtil.waitReplication(fs, sourcePath, (short) 1); Map counters = new HashMap(); refreshPolicy(); for (int i = 0; i < parityBlocks.size(); i++) { companionBlocks = getCompanionBlocks(namesystem, policy, parityBlocks.get(i).getBlock()); counters = BlockPlacementPolicyRaid.countCompanionBlocks(companionBlocks, false); Assert.assertTrue(counters.get(datanode1.getName()) >= 1 && counters.get(datanode1.getName()) <= 2); Assert.assertTrue(counters.get(datanode1.getName()) + counters.get(datanode2.getName()) == companionBlocks.size()); counters = BlockPlacementPolicyRaid.countCompanionBlocks(companionBlocks, true); Assert.assertTrue(counters.get(datanode1.getParent().getName()) >= 1 && counters.get(datanode1.getParent().getName()) <= 2); Assert.assertTrue(counters.get(datanode1.getParent().getName()) + counters.get(datanode2.getParent().getName()) == companionBlocks.size()); } } finally { if (cluster != null) { cluster.shutdown(); } }
5681 -1009651952apache/hadoopTsz-wo Szecabec474e0f4c525ba80bd7c8dede1a8d76e4f39 // Set the policy to default policy to place the block in the default way setBlockPlacementPolicy(namesystem, new BlockPlacementPolicyDefault( Set the policy to default policy to place the block in the default way SATD_CHANGED testDeleteReplica() public void testDeleteReplica() throws IOException setupCluster(); try { // Set the policy to default policy to place the block in the default way setBlockPlacementPolicy(namesystem, new BlockPlacementPolicyDefault(conf, namesystem, namesystem.clusterMap)); DatanodeDescriptor datanode1 = NameNodeRaidTestUtil.getDatanodeMap(namesystem).values().iterator().next(); String source = "/dir/file"; String parity = xorPrefix + source; final Path parityPath = new Path(parity); DFSTestUtil.createFile(fs, parityPath, 3, (short) 1, 0L); DFSTestUtil.waitReplication(fs, parityPath, (short) 1); // start one more datanode cluster.startDataNodes(conf, 1, true, null, rack2, host2, null); DatanodeDescriptor datanode2 = null; for (DatanodeDescriptor d : NameNodeRaidTestUtil.getDatanodeMap(namesystem).values()) { if (!d.getName().equals(datanode1.getName())) { datanode2 = d; } } Assert.assertTrue(datanode2 != null); cluster.waitActive(); final Path sourcePath = new Path(source); DFSTestUtil.createFile(fs, sourcePath, 5, (short) 2, 0L); DFSTestUtil.waitReplication(fs, sourcePath, (short) 2); refreshPolicy(); Assert.assertEquals(parity, policy.getParityFile(source)); Assert.assertEquals(source, policy.getSourceFile(parity, xorPrefix)); List sourceBlocks = getBlocks(namesystem, source); List parityBlocks = getBlocks(namesystem, parity); Assert.assertEquals(5, sourceBlocks.size()); Assert.assertEquals(3, parityBlocks.size()); // verify the result of getCompanionBlocks() Collection companionBlocks; companionBlocks = getCompanionBlocks(namesystem, policy, sourceBlocks.get(0).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 0, 1 }, new int[] { 0 }); companionBlocks = getCompanionBlocks(namesystem, policy, sourceBlocks.get(1).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 0, 1 }, new int[] { 0 }); companionBlocks = getCompanionBlocks(namesystem, policy, sourceBlocks.get(2).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 2, 3 }, new int[] { 1 }); companionBlocks = getCompanionBlocks(namesystem, policy, sourceBlocks.get(3).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 2, 3 }, new int[] { 1 }); companionBlocks = getCompanionBlocks(namesystem, policy, sourceBlocks.get(4).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 4 }, new int[] { 2 }); companionBlocks = getCompanionBlocks(namesystem, policy, parityBlocks.get(0).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 0, 1 }, new int[] { 0 }); companionBlocks = getCompanionBlocks(namesystem, policy, parityBlocks.get(1).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 2, 3 }, new int[] { 1 }); companionBlocks = getCompanionBlocks(namesystem, policy, parityBlocks.get(2).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 4 }, new int[] { 2 }); // Set the policy back to raid policy. We have to create a new object // here to clear the block location cache refreshPolicy(); setBlockPlacementPolicy(namesystem, policy); // verify policy deletes the correct blocks. companion blocks should be // evenly distributed. fs.setReplication(sourcePath, (short) 1); DFSTestUtil.waitReplication(fs, sourcePath, (short) 1); Map counters = new HashMap(); refreshPolicy(); for (int i = 0; i < parityBlocks.size(); i++) { companionBlocks = getCompanionBlocks(namesystem, policy, parityBlocks.get(i).getBlock()); counters = BlockPlacementPolicyRaid.countCompanionBlocks(companionBlocks, false); Assert.assertTrue(counters.get(datanode1.getName()) >= 1 && counters.get(datanode1.getName()) <= 2); Assert.assertTrue(counters.get(datanode1.getName()) + counters.get(datanode2.getName()) == companionBlocks.size()); counters = BlockPlacementPolicyRaid.countCompanionBlocks(companionBlocks, true); Assert.assertTrue(counters.get(datanode1.getParent().getName()) >= 1 && counters.get(datanode1.getParent().getName()) <= 2); Assert.assertTrue(counters.get(datanode1.getParent().getName()) + counters.get(datanode2.getParent().getName()) == companionBlocks.size()); } } finally { if (cluster != null) { cluster.shutdown(); } }
5679 -1009651953apache/hadoopTsz-wo Szecabec474e0f4c525ba80bd7c8dede1a8d76e4f39 // Set the policy to default policy to place the block in the default way setBlockPlacementPolicy(namesystem, new BlockPlacementPolicyDefault( Set the policy to default policy to place the block in the default way SATD_CHANGED testGetCompanionBLocks() public void testGetCompanionBLocks() throws IOException setupCluster(); try { String file1 = "/dir/file1"; String file2 = "/raid/dir/file2"; String file3 = "/raidrs/dir/file3"; // Set the policy to default policy to place the block in the default way setBlockPlacementPolicy(namesystem, new BlockPlacementPolicyDefault(conf, namesystem, namesystem.clusterMap)); DFSTestUtil.createFile(fs, new Path(file1), 3, (short) 1, 0L); DFSTestUtil.createFile(fs, new Path(file2), 4, (short) 1, 0L); DFSTestUtil.createFile(fs, new Path(file3), 8, (short) 1, 0L); Collection companionBlocks; companionBlocks = getCompanionBlocks(namesystem, policy, getBlocks(namesystem, file1).get(0).getBlock()); Assert.assertTrue(companionBlocks == null || companionBlocks.size() == 0); companionBlocks = getCompanionBlocks(namesystem, policy, getBlocks(namesystem, file1).get(2).getBlock()); Assert.assertTrue(companionBlocks == null || companionBlocks.size() == 0); companionBlocks = getCompanionBlocks(namesystem, policy, getBlocks(namesystem, file2).get(0).getBlock()); Assert.assertEquals(1, companionBlocks.size()); companionBlocks = getCompanionBlocks(namesystem, policy, getBlocks(namesystem, file2).get(3).getBlock()); Assert.assertEquals(1, companionBlocks.size()); int rsParityLength = RaidNode.rsParityLength(conf); companionBlocks = getCompanionBlocks(namesystem, policy, getBlocks(namesystem, file3).get(0).getBlock()); Assert.assertEquals(rsParityLength, companionBlocks.size()); companionBlocks = getCompanionBlocks(namesystem, policy, getBlocks(namesystem, file3).get(4).getBlock()); Assert.assertEquals(rsParityLength, companionBlocks.size()); companionBlocks = getCompanionBlocks(namesystem, policy, getBlocks(namesystem, file3).get(6).getBlock()); Assert.assertEquals(2, companionBlocks.size()); } finally { if (cluster != null) { cluster.shutdown(); } }
5677 -1009651954apache/hadoopTsz-wo Szecabec474e0f4c525ba80bd7c8dede1a8d76e4f39 // Set the policy to default policy to place the block in the default way setBlockPlacementPolicy(namesystem, new BlockPlacementPolicyDefault( test full path cache SATD_REMOVED testCachedPathNames() public void testCachedPathNames() throws IOException setupCluster(); try { String file1 = "/dir/file1"; String file2 = "/dir/file2"; DFSTestUtil.createFile(fs, new Path(file1), 3, (short) 1, 0L); DFSTestUtil.createFile(fs, new Path(file2), 4, (short) 1, 0L); // test full path cache CachedFullPathNames cachedFullPathNames = new CachedFullPathNames(namesystem); FSInodeInfo inode1 = null; FSInodeInfo inode2 = null; NameNodeRaidTestUtil.readLock(namesystem.dir); try { inode1 = NameNodeRaidTestUtil.getNode(namesystem.dir, file1, true); inode2 = NameNodeRaidTestUtil.getNode(namesystem.dir, file2, true); } finally { NameNodeRaidTestUtil.readUnLock(namesystem.dir); } verifyCachedFullPathNameResult(cachedFullPathNames, inode1); verifyCachedFullPathNameResult(cachedFullPathNames, inode1); verifyCachedFullPathNameResult(cachedFullPathNames, inode2); verifyCachedFullPathNameResult(cachedFullPathNames, inode2); try { Thread.sleep(1200L); } catch (InterruptedException e) { } verifyCachedFullPathNameResult(cachedFullPathNames, inode2); verifyCachedFullPathNameResult(cachedFullPathNames, inode1); } finally { if (cluster != null) { cluster.shutdown(); } }
5675 -1009651955apache/hadoopTsz-wo Szecabec474e0f4c525ba80bd7c8dede1a8d76e4f39 // Set the policy to default policy to place the block in the default way setBlockPlacementPolicy(namesystem, new BlockPlacementPolicyDefault( test blocks cache SATD_REMOVED testCachedBlocks() public void testCachedBlocks() throws IOException setupCluster(); try { String file1 = "/dir/file1"; String file2 = "/dir/file2"; DFSTestUtil.createFile(fs, new Path(file1), 3, (short) 1, 0L); DFSTestUtil.createFile(fs, new Path(file2), 4, (short) 1, 0L); // test blocks cache CachedLocatedBlocks cachedBlocks = new CachedLocatedBlocks(namesystem); verifyCachedBlocksResult(cachedBlocks, namesystem, file1); verifyCachedBlocksResult(cachedBlocks, namesystem, file1); verifyCachedBlocksResult(cachedBlocks, namesystem, file2); verifyCachedBlocksResult(cachedBlocks, namesystem, file2); try { Thread.sleep(1200L); } catch (InterruptedException e) { } verifyCachedBlocksResult(cachedBlocks, namesystem, file2); verifyCachedBlocksResult(cachedBlocks, namesystem, file1); } finally { if (cluster != null) { cluster.shutdown(); } }
5673 -1009651956apache/hadoopTsz-wo Szecabec474e0f4c525ba80bd7c8dede1a8d76e4f39 // Set the policy to default policy to place the block in the default way setBlockPlacementPolicy(namesystem, new BlockPlacementPolicyDefault( verify that every adjacent 4 blocks are on differnt nodes SATD_REMOVED testChooseTargetForHarRaidFile() public void testChooseTargetForHarRaidFile() throws IOException setupCluster(); try { String[] racks = { "/rack2", "/rack2", "/rack2", "/rack2", "/rack2", "/rack2" }; String[] hosts = { "host2.rack2.com", "host3.rack2.com", "host4.rack2.com", "host5.rack2.com", "host6.rack2.com", "host7.rack2.com" }; cluster.startDataNodes(conf, 6, true, null, racks, hosts, null); String harParity = raidrsHarTempPrefix + "/dir/file"; int numBlocks = 11; DFSTestUtil.createFile(fs, new Path(harParity), numBlocks, (short) 1, 0L); DFSTestUtil.waitReplication(fs, new Path(harParity), (short) 1); FileStatus stat = fs.getFileStatus(new Path(harParity)); BlockLocation[] loc = fs.getFileBlockLocations(stat, 0, stat.getLen()); int rsParityLength = RaidNode.rsParityLength(conf); for (int i = 0; i < numBlocks - rsParityLength; i++) { Set locations = new HashSet(); for (int j = 0; j < rsParityLength; j++) { for (int k = 0; k < loc[i + j].getNames().length; k++) { // verify that every adjacent 4 blocks are on differnt nodes String name = loc[i + j].getNames()[k]; LOG.info("Har Raid block location: " + name); Assert.assertTrue(locations.add(name)); } } } } finally { if (cluster != null) { cluster.shutdown(); } }
5671 -1009651957apache/hadoopTsz-wo Szecabec474e0f4c525ba80bd7c8dede1a8d76e4f39 // Set the policy to default policy to place the block in the default way setBlockPlacementPolicy(namesystem, new BlockPlacementPolicyDefault( start 3 more datanodes SATD_REMOVED testChooseTargetForRaidFile() public void testChooseTargetForRaidFile() throws IOException setupCluster(); try { String src = "/dir/file"; String parity = raidrsTempPrefix + src; DFSTestUtil.createFile(fs, new Path(src), 4, (short) 1, 0L); DFSTestUtil.waitReplication(fs, new Path(src), (short) 1); refreshPolicy(); setBlockPlacementPolicy(namesystem, policy); // start 3 more datanodes String[] racks = { "/rack2", "/rack2", "/rack2", "/rack2", "/rack2", "/rack2" }; String[] hosts = { "host2.rack2.com", "host3.rack2.com", "host4.rack2.com", "host5.rack2.com", "host6.rack2.com", "host7.rack2.com" }; cluster.startDataNodes(conf, 6, true, null, racks, hosts, null); int numBlocks = 6; DFSTestUtil.createFile(fs, new Path(parity), numBlocks, (short) 2, 0L); DFSTestUtil.waitReplication(fs, new Path(parity), (short) 2); FileStatus srcStat = fs.getFileStatus(new Path(src)); BlockLocation[] srcLoc = fs.getFileBlockLocations(srcStat, 0, srcStat.getLen()); FileStatus parityStat = fs.getFileStatus(new Path(parity)); BlockLocation[] parityLoc = fs.getFileBlockLocations(parityStat, 0, parityStat.getLen()); int parityLen = RaidNode.rsParityLength(conf); for (int i = 0; i < numBlocks / parityLen; i++) { Set locations = new HashSet(); for (int j = 0; j < srcLoc.length; j++) { String[] names = srcLoc[j].getNames(); for (int k = 0; k < names.length; k++) { LOG.info("Source block location: " + names[k]); locations.add(names[k]); } } for (int j = 0; j < parityLen; j++) { String[] names = parityLoc[j + i * parityLen].getNames(); for (int k = 0; k < names.length; k++) { LOG.info("Parity block location: " + names[k]); Assert.assertTrue(locations.add(names[k])); } } } } finally { if (cluster != null) { cluster.shutdown(); } }
5669 -1009651987apache/hadoopTsz-wo Szecabec474e0f4c525ba80bd7c8dede1a8d76e4f39 // Set the policy to default policy to place the block in the default way setBlockPlacementPolicy(namesystem, new BlockPlacementPolicyDefault( start the cluster with one datanode first SATD_REMOVED setupCluster() protected void setupCluster() throws IOException conf = new Configuration(); conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 1000L); conf.set("dfs.replication.pending.timeout.sec", "2"); conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 1L); conf.set("dfs.block.replicator.classname", "org.apache.hadoop.hdfs.server.namenode.BlockPlacementPolicyRaid"); conf.set(RaidNode.STRIPE_LENGTH_KEY, "2"); conf.set(RaidNode.RS_PARITY_LENGTH_KEY, "3"); conf.setInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, 1); // start the cluster with one datanode first cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).format(true).racks(rack1).hosts(host1).build(); cluster.waitActive(); namesystem = cluster.getNameNode().getNamesystem(); Assert.assertTrue("BlockPlacementPolicy type is not correct.", namesystem.blockManager.replicator instanceof BlockPlacementPolicyRaid); policy = (BlockPlacementPolicyRaid) namesystem.blockManager.replicator; fs = cluster.getFileSystem(); xorPrefix = RaidNode.xorDestinationPath(conf).toUri().getPath(); raidTempPrefix = RaidNode.xorTempPrefix(conf); raidrsTempPrefix = RaidNode.rsTempPrefix(conf); raidrsHarTempPrefix = RaidNode.rsHarTempPrefix(conf);
5642 -1009652111apache/hadoopSuresh Srinivas710e5a960e8af1d4c73e386041096aacfee8b828 Since we're creating a new UserGroupInformation here, we know that no future RPC proxies will be able to re-use the same connection. And usages of this proxy tend to be one-off calls. This is a temporary fix: callers should really achieve this by using RPC.stopProxy() on the resulting object, but this is currently not working in trunk. See the discussion on HDFS-1965. Since we're creating a new UserGroupInformation here, we know that no future RPC proxies will be able to re-use the same connection. And usages of this proxy tend to be one-off calls. This is a temporary fix: callers should really achieve this by using RPC.stopProxy() on the resulting object, but this is currently not working in trunk. See the discussion on HDFS-1965. SATD_MOVED_FILE createClientDatanodeProtocolProxy(DatanodeID, Configuration, int, LocatedBlock) public static ClientDatanodeProtocol createClientDatanodeProtocolProxy(DatanodeID datanodeid, Configuration conf, int socketTimeout, LocatedBlock locatedBlock) throws IOException InetSocketAddress addr = NetUtils.createSocketAddr(datanodeid.getHost() + ":" + datanodeid.getIpcPort()); if (ClientDatanodeProtocol.LOG.isDebugEnabled()) { ClientDatanodeProtocol.LOG.debug("ClientDatanodeProtocol addr=" + addr); } // Since we're creating a new UserGroupInformation here, we know that no // future RPC proxies will be able to re-use the same connection. And // usages of this proxy tend to be one-off calls. // // This is a temporary fix: callers should really achieve this by using // RPC.stopProxy() on the resulting object, but this is currently not // working in trunk. See the discussion on HDFS-1965. Configuration confWithNoIpcIdle = new Configuration(conf); confWithNoIpcIdle.setInt(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECTION_MAXIDLETIME_KEY, 0); UserGroupInformation ticket = UserGroupInformation.createRemoteUser(locatedBlock.getBlock().getLocalBlock().toString()); ticket.addToken(locatedBlock.getBlockToken()); return (ClientDatanodeProtocol) RPC.getProxy(ClientDatanodeProtocol.class, ClientDatanodeProtocol.versionID, addr, ticket, confWithNoIpcIdle, NetUtils.getDefaultSocketFactory(conf), socketTimeout);
5641 -1009652112apache/hadoopEli Collinsc3f6575ca44e8ad803d0b46991472465b595cdeb Move decommissioned datanodes to the bottom Move decommissioned datanodes to the bottom SATD_MOVED_FILE sortLocatedBlocks(String, List) public void sortLocatedBlocks(final String targethost, final List locatedblocks) // sort the blocks final DatanodeDescriptor client = getDatanodeByHost(targethost); for (LocatedBlock b : locatedblocks) { networktopology.pseudoSortByDistance(client, b.getLocations()); // Move decommissioned datanodes to the bottom Arrays.sort(b.getLocations(), DFSUtil.DECOM_COMPARATOR); }
5640 -1009652113apache/hadoopArun Murthy714edd65acb8d78640ea062e84ffe83c1e363738 None Exception while writing to the client. Connection closure from the other end is mostly the case and we do not care much about it. But other things can go wrong, especially in transferTo(), which we do not want to ignore. The message parsing below should not be considered as a good coding example. NEVER do it to drive a program logic. NEVER. It was done here because the NIO throws an IOException for EPIPE. SATD_ADDED sendChunks(ByteBuffer, int, OutputStream) private int sendChunks(ByteBuffer pkt, int maxChunks, OutputStream out) throws IOException // Sends multiple chunks in one packet with a single write(). int len = (int) Math.min(endOffset - offset, (((long) bytesPerChecksum) * ((long) maxChunks))); int numChunks = (len + bytesPerChecksum - 1) / bytesPerChecksum; int packetLen = len + numChunks * checksumSize + 4; boolean lastDataPacket = offset + len == endOffset && len > 0; pkt.clear(); PacketHeader header = new PacketHeader(packetLen, offset, seqno, (len == 0), len); header.putInBuffer(pkt); int checksumOff = pkt.position(); int checksumLen = numChunks * checksumSize; byte[] buf = pkt.array(); if (checksumSize > 0 && checksumIn != null) { try { checksumIn.readFully(buf, checksumOff, checksumLen); } catch (IOException e) { LOG.warn(" Could not read or failed to veirfy checksum for data" + " at offset " + offset + " for block " + block + " got : " + StringUtils.stringifyException(e)); IOUtils.closeStream(checksumIn); checksumIn = null; if (corruptChecksumOk) { if (checksumOff < checksumLen) { // Just fill the array with zeros. Arrays.fill(buf, checksumOff, checksumLen, (byte) 0); } } else { throw e; } } // write in progress that we need to use to get last checksum if (lastDataPacket && lastChunkChecksum != null) { int start = checksumOff + checksumLen - checksumSize; byte[] updatedChecksum = lastChunkChecksum.getChecksum(); if (updatedChecksum != null) { System.arraycopy(updatedChecksum, 0, buf, start, checksumSize); } } } int dataOff = checksumOff + checksumLen; if (blockInPosition < 0) { // normal transfer IOUtils.readFully(blockIn, buf, dataOff, len); if (verifyChecksum) { int dOff = dataOff; int cOff = checksumOff; int dLeft = len; for (int i = 0; i < numChunks; i++) { checksum.reset(); int dLen = Math.min(dLeft, bytesPerChecksum); checksum.update(buf, dOff, dLen); if (!checksum.compare(buf, cOff)) { long failedPos = offset + len - dLeft; throw new ChecksumException("Checksum failed at " + failedPos, failedPos); } dLeft -= dLen; dOff += dLen; cOff += checksumSize; } } // writing is done below (mainly to handle IOException) } try { if (blockInPosition >= 0) { // use transferTo(). Checks on out and blockIn are already done. SocketOutputStream sockOut = (SocketOutputStream) out; // first write the packet sockOut.write(buf, 0, dataOff); // no need to flush. since we know out is not a buffered stream. sockOut.transferToFully(((FileInputStream) blockIn).getChannel(), blockInPosition, len); blockInPosition += len; } else { // normal transfer out.write(buf, 0, dataOff + len); } } catch (IOException e) { /* Exception while writing to the client. Connection closure from * the other end is mostly the case and we do not care much about * it. But other things can go wrong, especially in transferTo(), * which we do not want to ignore. * * The message parsing below should not be considered as a good * coding example. NEVER do it to drive a program logic. NEVER. * It was done here because the NIO throws an IOException for EPIPE. */ String ioem = e.getMessage(); if (!ioem.startsWith("Broken pipe") && !ioem.startsWith("Connection reset")) { LOG.error("BlockSender.sendChunks() exception: ", e); } throw ioeToSocketException(e); } if (throttler != null) { // rebalancing so throttle throttler.throttle(packetLen); } return len;
5639 -1009652114apache/hadoopAaron Myers224972e0558e7a0022002ed26e765acdfd9f8f6c None The following XDR recipe was done through a careful reading of gm_protocol.x in Ganglia 3.1 and carefully examining the output of the gmetric utility with strace. SATD_ADDED emitMetric(String, String, String, String, GangliaConf, GangliaSlope) protected void emitMetric(String groupName, String name, String type, String value, GangliaConf gConf, GangliaSlope gSlope) throws IOException if (name == null) { LOG.warn("Metric was emitted with no name."); return; } else if (value == null) { LOG.warn("Metric name " + name + " was emitted with a null value."); return; } else if (type == null) { LOG.warn("Metric name " + name + ", value " + value + " has no type."); return; } if (LOG.isDebugEnabled()) { LOG.debug("Emitting metric " + name + ", type " + type + ", value " + value + ", slope " + gSlope.name() + " from hostname " + getHostName()); } // The following XDR recipe was done through a careful reading of // gm_protocol.x in Ganglia 3.1 and carefully examining the output of // the gmetric utility with strace. // First we send out a metadata message // metric_id = metadata_msg xdr_int(128); // hostname xdr_string(getHostName()); // metric name xdr_string(name); // spoof = False xdr_int(0); // metric type xdr_string(type); // metric name xdr_string(name); // units xdr_string(gConf.getUnits()); // slope xdr_int(gSlope.ordinal()); // tmax, the maximum time between metrics xdr_int(gConf.getTmax()); // dmax, the maximum data value xdr_int(gConf.getDmax()); xdr_int(1); /*Num of the entries in extra_value field for Ganglia 3.1.x*/ xdr_string("GROUP"); /*Group attribute*/ xdr_string(groupName); /*Group value*/ // send the metric to Ganglia hosts emitToGangliaHosts(); // Now we send out a message with the actual value. // Technically, we only need to send out the metadata message once for // each metric, but I don't want to have to record which metrics we did and // did not send. // we are sending a string value xdr_int(133); // hostName xdr_string(getHostName()); // metric name xdr_string(name); // spoof = False xdr_int(0); // format field xdr_string("%s"); // metric value xdr_string(value); // send the metric to Ganglia hosts emitToGangliaHosts();
5637 -1009652115apache/hadoopTodd Lipconff0511019cd75a113a10940ef09b0d141a77c91a None TODO: update after fixing HADOOP-7352 SATD_ADDED testListStatusThrowsExceptionForUnreadableDir() public void testListStatusThrowsExceptionForUnreadableDir() throws Exception Path testRootDir = getTestRootPath(fSys, "test/hadoop/dir"); Path obscuredDir = new Path(testRootDir, "foo"); // so foo is non-empty Path subDir = new Path(obscuredDir, "bar"); fSys.mkdirs(subDir); // no access fSys.setPermission(obscuredDir, new FsPermission((short) 0)); try { fSys.listStatus(obscuredDir); Assert.fail("Should throw IOException"); } catch (IOException ioe) { // expected } finally { // make sure the test directory can be deleted // default fSys.setPermission(obscuredDir, new FsPermission((short) 0755)); }
5636 -1009652166apache/hadoopTsz-wo Sze61fa4153dc859e19529aea8f24762f6ea2dd2ee2 verify policy deletes the correct blocks. companion blocks should be evenly distributed. verify policy deletes the correct blocks. companion blocks should be evenly distributed. FILE_PATH_CHANGED testDeleteReplica() public void testDeleteReplica() throws IOException setupCluster(); try { // Set the policy to default policy to place the block in the default way setBlockPlacementPolicy(namesystem, new BlockPlacementPolicyDefault(conf, namesystem, namesystem.clusterMap)); DatanodeDescriptor datanode1 = NameNodeRaidTestUtil.getDatanodeMap(namesystem).values().iterator().next(); String source = "/dir/file"; String parity = xorPrefix + source; final Path parityPath = new Path(parity); DFSTestUtil.createFile(fs, parityPath, 3, (short) 1, 0L); DFSTestUtil.waitReplication(fs, parityPath, (short) 1); // start one more datanode cluster.startDataNodes(conf, 1, true, null, rack2, host2, null); DatanodeDescriptor datanode2 = null; for (DatanodeDescriptor d : NameNodeRaidTestUtil.getDatanodeMap(namesystem).values()) { if (!d.getName().equals(datanode1.getName())) { datanode2 = d; } } Assert.assertTrue(datanode2 != null); cluster.waitActive(); final Path sourcePath = new Path(source); DFSTestUtil.createFile(fs, sourcePath, 5, (short) 2, 0L); DFSTestUtil.waitReplication(fs, sourcePath, (short) 2); refreshPolicy(); Assert.assertEquals(parity, policy.getParityFile(source)); Assert.assertEquals(source, policy.getSourceFile(parity, xorPrefix)); List sourceBlocks = getBlocks(namesystem, source); List parityBlocks = getBlocks(namesystem, parity); Assert.assertEquals(5, sourceBlocks.size()); Assert.assertEquals(3, parityBlocks.size()); // verify the result of getCompanionBlocks() Collection companionBlocks; companionBlocks = getCompanionBlocks(namesystem, policy, sourceBlocks.get(0).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 0, 1 }, new int[] { 0 }); companionBlocks = getCompanionBlocks(namesystem, policy, sourceBlocks.get(1).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 0, 1 }, new int[] { 0 }); companionBlocks = getCompanionBlocks(namesystem, policy, sourceBlocks.get(2).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 2, 3 }, new int[] { 1 }); companionBlocks = getCompanionBlocks(namesystem, policy, sourceBlocks.get(3).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 2, 3 }, new int[] { 1 }); companionBlocks = getCompanionBlocks(namesystem, policy, sourceBlocks.get(4).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 4 }, new int[] { 2 }); companionBlocks = getCompanionBlocks(namesystem, policy, parityBlocks.get(0).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 0, 1 }, new int[] { 0 }); companionBlocks = getCompanionBlocks(namesystem, policy, parityBlocks.get(1).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 2, 3 }, new int[] { 1 }); companionBlocks = getCompanionBlocks(namesystem, policy, parityBlocks.get(2).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 4 }, new int[] { 2 }); // Set the policy back to raid policy. We have to create a new object // here to clear the block location cache refreshPolicy(); setBlockPlacementPolicy(namesystem, policy); // verify policy deletes the correct blocks. companion blocks should be // evenly distributed. fs.setReplication(sourcePath, (short) 1); DFSTestUtil.waitReplication(fs, sourcePath, (short) 1); Map counters = new HashMap(); refreshPolicy(); for (int i = 0; i < parityBlocks.size(); i++) { companionBlocks = getCompanionBlocks(namesystem, policy, parityBlocks.get(i).getBlock()); counters = BlockPlacementPolicyRaid.countCompanionBlocks(companionBlocks, false); Assert.assertTrue(counters.get(datanode1.getName()) >= 1 && counters.get(datanode1.getName()) <= 2); Assert.assertTrue(counters.get(datanode1.getName()) + counters.get(datanode2.getName()) == companionBlocks.size()); counters = BlockPlacementPolicyRaid.countCompanionBlocks(companionBlocks, true); Assert.assertTrue(counters.get(datanode1.getParent().getName()) >= 1 && counters.get(datanode1.getParent().getName()) <= 2); Assert.assertTrue(counters.get(datanode1.getParent().getName()) + counters.get(datanode2.getParent().getName()) == companionBlocks.size()); } } finally { if (cluster != null) { cluster.shutdown(); } }
5635 -1009652167apache/hadoopTsz-wo Sze61fa4153dc859e19529aea8f24762f6ea2dd2ee2 Set the policy to default policy to place the block in the default way Set the policy to default policy to place the block in the default way FILE_PATH_CHANGED testDeleteReplica() public void testDeleteReplica() throws IOException setupCluster(); try { // Set the policy to default policy to place the block in the default way setBlockPlacementPolicy(namesystem, new BlockPlacementPolicyDefault(conf, namesystem, namesystem.clusterMap)); DatanodeDescriptor datanode1 = NameNodeRaidTestUtil.getDatanodeMap(namesystem).values().iterator().next(); String source = "/dir/file"; String parity = xorPrefix + source; final Path parityPath = new Path(parity); DFSTestUtil.createFile(fs, parityPath, 3, (short) 1, 0L); DFSTestUtil.waitReplication(fs, parityPath, (short) 1); // start one more datanode cluster.startDataNodes(conf, 1, true, null, rack2, host2, null); DatanodeDescriptor datanode2 = null; for (DatanodeDescriptor d : NameNodeRaidTestUtil.getDatanodeMap(namesystem).values()) { if (!d.getName().equals(datanode1.getName())) { datanode2 = d; } } Assert.assertTrue(datanode2 != null); cluster.waitActive(); final Path sourcePath = new Path(source); DFSTestUtil.createFile(fs, sourcePath, 5, (short) 2, 0L); DFSTestUtil.waitReplication(fs, sourcePath, (short) 2); refreshPolicy(); Assert.assertEquals(parity, policy.getParityFile(source)); Assert.assertEquals(source, policy.getSourceFile(parity, xorPrefix)); List sourceBlocks = getBlocks(namesystem, source); List parityBlocks = getBlocks(namesystem, parity); Assert.assertEquals(5, sourceBlocks.size()); Assert.assertEquals(3, parityBlocks.size()); // verify the result of getCompanionBlocks() Collection companionBlocks; companionBlocks = getCompanionBlocks(namesystem, policy, sourceBlocks.get(0).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 0, 1 }, new int[] { 0 }); companionBlocks = getCompanionBlocks(namesystem, policy, sourceBlocks.get(1).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 0, 1 }, new int[] { 0 }); companionBlocks = getCompanionBlocks(namesystem, policy, sourceBlocks.get(2).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 2, 3 }, new int[] { 1 }); companionBlocks = getCompanionBlocks(namesystem, policy, sourceBlocks.get(3).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 2, 3 }, new int[] { 1 }); companionBlocks = getCompanionBlocks(namesystem, policy, sourceBlocks.get(4).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 4 }, new int[] { 2 }); companionBlocks = getCompanionBlocks(namesystem, policy, parityBlocks.get(0).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 0, 1 }, new int[] { 0 }); companionBlocks = getCompanionBlocks(namesystem, policy, parityBlocks.get(1).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 2, 3 }, new int[] { 1 }); companionBlocks = getCompanionBlocks(namesystem, policy, parityBlocks.get(2).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 4 }, new int[] { 2 }); // Set the policy back to raid policy. We have to create a new object // here to clear the block location cache refreshPolicy(); setBlockPlacementPolicy(namesystem, policy); // verify policy deletes the correct blocks. companion blocks should be // evenly distributed. fs.setReplication(sourcePath, (short) 1); DFSTestUtil.waitReplication(fs, sourcePath, (short) 1); Map counters = new HashMap(); refreshPolicy(); for (int i = 0; i < parityBlocks.size(); i++) { companionBlocks = getCompanionBlocks(namesystem, policy, parityBlocks.get(i).getBlock()); counters = BlockPlacementPolicyRaid.countCompanionBlocks(companionBlocks, false); Assert.assertTrue(counters.get(datanode1.getName()) >= 1 && counters.get(datanode1.getName()) <= 2); Assert.assertTrue(counters.get(datanode1.getName()) + counters.get(datanode2.getName()) == companionBlocks.size()); counters = BlockPlacementPolicyRaid.countCompanionBlocks(companionBlocks, true); Assert.assertTrue(counters.get(datanode1.getParent().getName()) >= 1 && counters.get(datanode1.getParent().getName()) <= 2); Assert.assertTrue(counters.get(datanode1.getParent().getName()) + counters.get(datanode2.getParent().getName()) == companionBlocks.size()); } } finally { if (cluster != null) { cluster.shutdown(); } }
5634 -1009652168apache/hadoopTsz-wo Sze61fa4153dc859e19529aea8f24762f6ea2dd2ee2 Set the policy to default policy to place the block in the default way Set the policy to default policy to place the block in the default way FILE_PATH_CHANGED testGetCompanionBLocks() public void testGetCompanionBLocks() throws IOException setupCluster(); try { String file1 = "/dir/file1"; String file2 = "/raid/dir/file2"; String file3 = "/raidrs/dir/file3"; // Set the policy to default policy to place the block in the default way setBlockPlacementPolicy(namesystem, new BlockPlacementPolicyDefault(conf, namesystem, namesystem.clusterMap)); DFSTestUtil.createFile(fs, new Path(file1), 3, (short) 1, 0L); DFSTestUtil.createFile(fs, new Path(file2), 4, (short) 1, 0L); DFSTestUtil.createFile(fs, new Path(file3), 8, (short) 1, 0L); Collection companionBlocks; companionBlocks = getCompanionBlocks(namesystem, policy, getBlocks(namesystem, file1).get(0).getBlock()); Assert.assertTrue(companionBlocks == null || companionBlocks.size() == 0); companionBlocks = getCompanionBlocks(namesystem, policy, getBlocks(namesystem, file1).get(2).getBlock()); Assert.assertTrue(companionBlocks == null || companionBlocks.size() == 0); companionBlocks = getCompanionBlocks(namesystem, policy, getBlocks(namesystem, file2).get(0).getBlock()); Assert.assertEquals(1, companionBlocks.size()); companionBlocks = getCompanionBlocks(namesystem, policy, getBlocks(namesystem, file2).get(3).getBlock()); Assert.assertEquals(1, companionBlocks.size()); int rsParityLength = RaidNode.rsParityLength(conf); companionBlocks = getCompanionBlocks(namesystem, policy, getBlocks(namesystem, file3).get(0).getBlock()); Assert.assertEquals(rsParityLength, companionBlocks.size()); companionBlocks = getCompanionBlocks(namesystem, policy, getBlocks(namesystem, file3).get(4).getBlock()); Assert.assertEquals(rsParityLength, companionBlocks.size()); companionBlocks = getCompanionBlocks(namesystem, policy, getBlocks(namesystem, file3).get(6).getBlock()); Assert.assertEquals(2, companionBlocks.size()); } finally { if (cluster != null) { cluster.shutdown(); } }
5633 -1009652116apache/hadoopTsz-wo Sze09b6f98de431628c80bc8a6faf0070eeaf72ff2a None Keeps a TreeSet for every named node. Each treeset contains a list of the blocks that are \"extra\" at that location. We'll eventually remove these extras. Mapping: StorageID -> TreeSet SATD_ADDED getPendingReplicationBlocksCount() public long getPendingReplicationBlocksCount() return pendingReplicationBlocksCount;
5632 -1009652344apache/hadoopTsz-wo Sze09b6f98de431628c80bc8a6faf0070eeaf72ff2a returns an iterator of all blocks in a given priority queue returns an iterator of all blocks in a given priority queue FILE_PATH_CHANGED clear() void clear() for (int i = 0; i < LEVEL; i++) { priorityQueues.get(i).clear(); }
5631 -1009652345apache/hadoopTsz-wo Sze09b6f98de431628c80bc8a6faf0070eeaf72ff2a add a block to a under replication queue according to its priority @param block a under replication block @param curReplicas current number of replicas of the block @param expectedReplicas expected number of replicas of the block add a block to a under replication queue according to its priority @param block a under replication block @param curReplicas current number of replicas of the block @param expectedReplicas expected number of replicas of the block FILE_PATH_CHANGED clear() void clear() for (int i = 0; i < LEVEL; i++) { priorityQueues.get(i).clear(); }
5630 -1009652346apache/hadoopTsz-wo Sze09b6f98de431628c80bc8a6faf0070eeaf72ff2a Return the priority of a block @param block a under replication block @param curReplicas current number of replicas of the block @param expectedReplicas expected number of replicas of the block Return the priority of a block @param block a under replication block @param curReplicas current number of replicas of the block @param expectedReplicas expected number of replicas of the block FILE_PATH_CHANGED clear() void clear() for (int i = 0; i < LEVEL; i++) { priorityQueues.get(i).clear(); }
5629 -1009652203apache/hadoopTsz-wo Sze09b6f98de431628c80bc8a6faf0070eeaf72ff2a Sufficient to rely on super's implementation Sufficient to rely on super's implementation FILE_PATH_CHANGED equals(Object) public boolean equals(Object obj) // Sufficient to rely on super's implementation return (this == obj) || super.equals(obj);
5628 -1009652224apache/hadoopTsz-wo Sze09b6f98de431628c80bc8a6faf0070eeaf72ff2a BlockInfo BlockInfoUnderConstruction participates in maps the same way as BlockInfo BlockInfo BlockInfoUnderConstruction participates in maps the same way as BlockInfo FILE_PATH_CHANGED hashCode() public int hashCode() return super.hashCode();
5627 -1009652225apache/hadoopTsz-wo Sze09b6f98de431628c80bc8a6faf0070eeaf72ff2a Sufficient to rely on super's implementation Sufficient to rely on super's implementation FILE_PATH_CHANGED equals(Object) public boolean equals(Object obj) // Sufficient to rely on super's implementation return (this == obj) || super.equals(obj);
5626 -1009652349apache/hadoopTsz-wo Sze09b6f98de431628c80bc8a6faf0070eeaf72ff2a the block is only in one of the to-do lists if it is in none then data-node already has it the block is only in one of the to-do lists if it is in none then data-node already has it FILE_PATH_CHANGED addBlock(DatanodeDescriptor, Block, String) public void addBlock(DatanodeDescriptor node, Block block, String delHint) throws IOException // decrement number of blocks scheduled to this datanode. node.decBlocksScheduled(); // get the deletion hint node DatanodeDescriptor delHintNode = null; if (delHint != null && delHint.length() != 0) { delHintNode = namesystem.getDatanode(delHint); if (delHintNode == null) { NameNode.stateChangeLog.warn("BLOCK* NameSystem.blockReceived: " + block + " is expected to be removed from an unrecorded node " + delHint); } } // // Modify the blocks->datanode map and node's map. // pendingReplications.remove(block); // blockReceived reports a finalized block Collection toAdd = new LinkedList(); Collection toInvalidate = new LinkedList(); Collection toCorrupt = new LinkedList(); Collection toUC = new LinkedList(); processReportedBlock(node, block, ReplicaState.FINALIZED, toAdd, toInvalidate, toCorrupt, toUC); // the block is only in one of the to-do lists // if it is in none then data-node already has it assert toUC.size() + toAdd.size() + toInvalidate.size() + toCorrupt.size() <= 1 : "The block should be only in one of the lists."; for (StatefulBlockInfo b : toUC) { addStoredBlockUnderConstruction(b.storedBlock, node, b.reportedState); } for (BlockInfo b : toAdd) { addStoredBlock(b, node, delHintNode, true); } for (Block b : toInvalidate) { NameNode.stateChangeLog.info("BLOCK* NameSystem.addBlock: block " + b + " on " + node.getName() + " size " + b.getNumBytes() + " does not belong to any file."); addToInvalidates(b, node); } for (BlockInfo b : toCorrupt) { markBlockAsCorrupt(b, node); }
5625 -1009652350apache/hadoopTsz-wo Sze09b6f98de431628c80bc8a6faf0070eeaf72ff2a The next two methods test the various cases under which we must conclude the replica is corrupt, or under construction. These are laid out as switch statements, on the theory that it is easier to understand the combinatorics of reportedState and ucState that way. It should be at least as efficient as boolean expressions. The next two methods test the various cases under which we must conclude the replica is corrupt, or under construction. These are laid out as switch statements, on the theory that it is easier to understand the combinatorics of reportedState and ucState that way. It should be at least as efficient as boolean expressions. FILE_PATH_CHANGED isReplicaCorrupt(Block, ReplicaState, BlockInfo, BlockUCState, DatanodeDescriptor) private boolean isReplicaCorrupt(Block iblk, ReplicaState reportedState, BlockInfo storedBlock, BlockUCState ucState, DatanodeDescriptor dn) switch(reportedState) { case FINALIZED: switch(ucState) { case COMPLETE: case COMMITTED: return (storedBlock.getGenerationStamp() != iblk.getGenerationStamp() || storedBlock.getNumBytes() != iblk.getNumBytes()); default: return false; } case RBW: case RWR: return storedBlock.isComplete(); // should not be reported case RUR: // should not be reported case TEMPORARY: default: FSNamesystem.LOG.warn("Unexpected replica state " + reportedState + " for block: " + storedBlock + " on " + dn.getName() + " size " + storedBlock.getNumBytes()); return true; }
5624 -1009652351apache/hadoopTsz-wo Sze09b6f98de431628c80bc8a6faf0070eeaf72ff2a move block to the head of the list move block to the head of the list FILE_PATH_CHANGED reportDiff(DatanodeDescriptor, BlockListAsLongs, Collection, Collection, Collection, Collection, Collection) void reportDiff(DatanodeDescriptor dn, BlockListAsLongs newReport, Collection toAdd, Collection toRemove, Collection toInvalidate, Collection toCorrupt, Collection toUC) // add to under-construction list // place a delimiter in the list which separates blocks // that have been reported from those that have not BlockInfo delimiter = new BlockInfo(new Block(), 1); boolean added = dn.addBlock(delimiter); assert added : "Delimiting block cannot be present in the node"; if (newReport == null) newReport = new BlockListAsLongs(); // scan the report and process newly reported blocks BlockReportIterator itBR = newReport.getBlockReportIterator(); while (itBR.hasNext()) { Block iblk = itBR.next(); ReplicaState iState = itBR.getCurrentReplicaState(); BlockInfo storedBlock = processReportedBlock(dn, iblk, iState, toAdd, toInvalidate, toCorrupt, toUC); // move block to the head of the list if (storedBlock != null && storedBlock.findDatanode(dn) >= 0) dn.moveBlockToHead(storedBlock); } // collect blocks that have not been reported // all of them are next to the delimiter Iterator it = new DatanodeDescriptor.BlockIterator(delimiter.getNext(0), dn); while (it.hasNext()) toRemove.add(it.next()); dn.removeBlock(delimiter);
5623 -1009652352apache/hadoopTsz-wo Sze09b6f98de431628c80bc8a6faf0070eeaf72ff2a Initial block reports can be processed a lot more efficiently than ordinary block reports. This shortens NN restart times. Initial block reports can be processed a lot more efficiently than ordinary block reports. This shortens NN restart times. FILE_PATH_CHANGED processReport(DatanodeDescriptor, BlockListAsLongs) public void processReport(DatanodeDescriptor node, BlockListAsLongs report) throws IOException boolean isFirstBlockReport = (node.numBlocks() == 0); if (isFirstBlockReport) { // Initial block reports can be processed a lot more efficiently than // ordinary block reports. This shortens NN restart times. processFirstBlockReport(node, report); return; } // Normal case: // Modify the (block-->datanode) map, according to the difference // between the old and new block report. // Collection toAdd = new LinkedList(); Collection toRemove = new LinkedList(); Collection toInvalidate = new LinkedList(); Collection toCorrupt = new LinkedList(); Collection toUC = new LinkedList(); reportDiff(node, report, toAdd, toRemove, toInvalidate, toCorrupt, toUC); // Process the blocks on each queue for (StatefulBlockInfo b : toUC) { addStoredBlockUnderConstruction(b.storedBlock, node, b.reportedState); } for (Block b : toRemove) { removeStoredBlock(b, node); } for (BlockInfo b : toAdd) { addStoredBlock(b, node, null, true); } for (Block b : toInvalidate) { NameNode.stateChangeLog.info("BLOCK* NameSystem.processReport: block " + b + " on " + node.getName() + " size " + b.getNumBytes() + " does not belong to any file."); addToInvalidates(b, node); } for (BlockInfo b : toCorrupt) { markBlockAsCorrupt(b, node); }
5619 -1009652349apache/hadoopTsz-wo Sze97b6ca4dd7d1233e8f8c90b1c01e47228c044e13 the block is only in one of the to-do lists if it is in none then data-node already has it the block is only in one of the to-do lists if it is in none then data-node already has it FILE_PATH_CHANGED addBlock(DatanodeDescriptor, Block, String) void addBlock(DatanodeDescriptor node, Block block, String delHint) throws IOException // decrement number of blocks scheduled to this datanode. node.decBlocksScheduled(); // get the deletion hint node DatanodeDescriptor delHintNode = null; if (delHint != null && delHint.length() != 0) { delHintNode = namesystem.getDatanode(delHint); if (delHintNode == null) { NameNode.stateChangeLog.warn("BLOCK* NameSystem.blockReceived: " + block + " is expected to be removed from an unrecorded node " + delHint); } } // // Modify the blocks->datanode map and node's map. // pendingReplications.remove(block); // blockReceived reports a finalized block Collection toAdd = new LinkedList(); Collection toInvalidate = new LinkedList(); Collection toCorrupt = new LinkedList(); Collection toUC = new LinkedList(); processReportedBlock(node, block, ReplicaState.FINALIZED, toAdd, toInvalidate, toCorrupt, toUC); // the block is only in one of the to-do lists // if it is in none then data-node already has it assert toUC.size() + toAdd.size() + toInvalidate.size() + toCorrupt.size() <= 1 : "The block should be only in one of the lists."; for (StatefulBlockInfo b : toUC) { addStoredBlockUnderConstruction(b.storedBlock, node, b.reportedState); } for (BlockInfo b : toAdd) { addStoredBlock(b, node, delHintNode, true); } for (Block b : toInvalidate) { NameNode.stateChangeLog.info("BLOCK* NameSystem.addBlock: block " + b + " on " + node.getName() + " size " + b.getNumBytes() + " does not belong to any file."); addToInvalidates(b, node); } for (BlockInfo b : toCorrupt) { markBlockAsCorrupt(b, node); }
5618 -1009652350apache/hadoopTsz-wo Sze97b6ca4dd7d1233e8f8c90b1c01e47228c044e13 The next two methods test the various cases under which we must conclude the replica is corrupt, or under construction. These are laid out as switch statements, on the theory that it is easier to understand the combinatorics of reportedState and ucState that way. It should be at least as efficient as boolean expressions. The next two methods test the various cases under which we must conclude the replica is corrupt, or under construction. These are laid out as switch statements, on the theory that it is easier to understand the combinatorics of reportedState and ucState that way. It should be at least as efficient as boolean expressions. FILE_PATH_CHANGED isReplicaCorrupt(Block, ReplicaState, BlockInfo, BlockUCState, DatanodeDescriptor) private boolean isReplicaCorrupt(Block iblk, ReplicaState reportedState, BlockInfo storedBlock, BlockUCState ucState, DatanodeDescriptor dn) switch(reportedState) { case FINALIZED: switch(ucState) { case COMPLETE: case COMMITTED: return (storedBlock.getGenerationStamp() != iblk.getGenerationStamp() || storedBlock.getNumBytes() != iblk.getNumBytes()); default: return false; } case RBW: case RWR: return storedBlock.isComplete(); // should not be reported case RUR: // should not be reported case TEMPORARY: default: FSNamesystem.LOG.warn("Unexpected replica state " + reportedState + " for block: " + storedBlock + " on " + dn.getName() + " size " + storedBlock.getNumBytes()); return true; }
5617 -1009652351apache/hadoopTsz-wo Sze97b6ca4dd7d1233e8f8c90b1c01e47228c044e13 move block to the head of the list move block to the head of the list FILE_PATH_CHANGED reportDiff(DatanodeDescriptor, BlockListAsLongs, Collection, Collection, Collection, Collection, Collection) void reportDiff(DatanodeDescriptor dn, BlockListAsLongs newReport, Collection toAdd, Collection toRemove, Collection toInvalidate, Collection toCorrupt, Collection toUC) // add to under-construction list // place a delimiter in the list which separates blocks // that have been reported from those that have not BlockInfo delimiter = new BlockInfo(new Block(), 1); boolean added = dn.addBlock(delimiter); assert added : "Delimiting block cannot be present in the node"; if (newReport == null) newReport = new BlockListAsLongs(); // scan the report and process newly reported blocks BlockReportIterator itBR = newReport.getBlockReportIterator(); while (itBR.hasNext()) { Block iblk = itBR.next(); ReplicaState iState = itBR.getCurrentReplicaState(); BlockInfo storedBlock = processReportedBlock(dn, iblk, iState, toAdd, toInvalidate, toCorrupt, toUC); // move block to the head of the list if (storedBlock != null && storedBlock.findDatanode(dn) >= 0) dn.moveBlockToHead(storedBlock); } // collect blocks that have not been reported // all of them are next to the delimiter Iterator it = new DatanodeDescriptor.BlockIterator(delimiter.getNext(0), dn); while (it.hasNext()) toRemove.add(it.next()); dn.removeBlock(delimiter);
5616 -1009652352apache/hadoopTsz-wo Sze97b6ca4dd7d1233e8f8c90b1c01e47228c044e13 Initial block reports can be processed a lot more efficiently than ordinary block reports. This shortens NN restart times. Initial block reports can be processed a lot more efficiently than ordinary block reports. This shortens NN restart times. FILE_PATH_CHANGED processReport(DatanodeDescriptor, BlockListAsLongs) public void processReport(DatanodeDescriptor node, BlockListAsLongs report) throws IOException boolean isFirstBlockReport = (node.numBlocks() == 0); if (isFirstBlockReport) { // Initial block reports can be processed a lot more efficiently than // ordinary block reports. This shortens NN restart times. processFirstBlockReport(node, report); return; } // Normal case: // Modify the (block-->datanode) map, according to the difference // between the old and new block report. // Collection toAdd = new LinkedList(); Collection toRemove = new LinkedList(); Collection toInvalidate = new LinkedList(); Collection toCorrupt = new LinkedList(); Collection toUC = new LinkedList(); reportDiff(node, report, toAdd, toRemove, toInvalidate, toCorrupt, toUC); // Process the blocks on each queue for (StatefulBlockInfo b : toUC) { addStoredBlockUnderConstruction(b.storedBlock, node, b.reportedState); } for (Block b : toRemove) { removeStoredBlock(b, node); } for (BlockInfo b : toAdd) { addStoredBlock(b, node, null, true); } for (Block b : toInvalidate) { NameNode.stateChangeLog.info("BLOCK* NameSystem.processReport: block " + b + " on " + node.getName() + " size " + b.getNumBytes() + " does not belong to any file."); addToInvalidates(b, node); } for (BlockInfo b : toCorrupt) { markBlockAsCorrupt(b, node); }
5613 -1009652224apache/hadoopTsz-wo Sze97b6ca4dd7d1233e8f8c90b1c01e47228c044e13 BlockInfo BlockInfoUnderConstruction participates in maps the same way as BlockInfo BlockInfo BlockInfoUnderConstruction participates in maps the same way as BlockInfo FILE_PATH_CHANGED hashCode() public int hashCode() return super.hashCode();
5612 -1009652225apache/hadoopTsz-wo Sze97b6ca4dd7d1233e8f8c90b1c01e47228c044e13 Sufficient to rely on super's implementation Sufficient to rely on super's implementation FILE_PATH_CHANGED equals(Object) public boolean equals(Object obj) // Sufficient to rely on super's implementation return (this == obj) || super.equals(obj);
5611 -1009652203apache/hadoopTsz-wo Sze97b6ca4dd7d1233e8f8c90b1c01e47228c044e13 Sufficient to rely on super's implementation Sufficient to rely on super's implementation FILE_PATH_CHANGED equals(Object) public boolean equals(Object obj) // Sufficient to rely on super's implementation return (this == obj) || super.equals(obj);
5610 -1009652344apache/hadoopTsz-wo Sze97b6ca4dd7d1233e8f8c90b1c01e47228c044e13 returns an iterator of all blocks in a given priority queue returns an iterator of all blocks in a given priority queue FILE_PATH_CHANGED clear() void clear() for (int i = 0; i < LEVEL; i++) { priorityQueues.get(i).clear(); }
5609 -1009652345apache/hadoopTsz-wo Sze97b6ca4dd7d1233e8f8c90b1c01e47228c044e13 add a block to a under replication queue according to its priority @param block a under replication block @param curReplicas current number of replicas of the block @param expectedReplicas expected number of replicas of the block add a block to a under replication queue according to its priority @param block a under replication block @param curReplicas current number of replicas of the block @param expectedReplicas expected number of replicas of the block FILE_PATH_CHANGED clear() void clear() for (int i = 0; i < LEVEL; i++) { priorityQueues.get(i).clear(); }
5608 -1009652346apache/hadoopTsz-wo Sze97b6ca4dd7d1233e8f8c90b1c01e47228c044e13 Return the priority of a block @param block a under replication block @param curReplicas current number of replicas of the block @param expectedReplicas expected number of replicas of the block Return the priority of a block @param block a under replication block @param curReplicas current number of replicas of the block @param expectedReplicas expected number of replicas of the block FILE_PATH_CHANGED clear() void clear() for (int i = 0; i < LEVEL; i++) { priorityQueues.get(i).clear(); }
5607 -1009652120apache/hadoopEli Collins1bcfe45e47775b98cce5541f328c4fd46e5eb13d None Keeps a TreeSet for every named node. Each treeset contains a list of the blocks that are \"extra\" at that location. We'll eventually remove these extras. Mapping: StorageID -> TreeSet SATD_ADDED getPendingReplicationBlocksCount() public long getPendingReplicationBlocksCount() return pendingReplicationBlocksCount;
5606 -1009652344apache/hadoopEli Collins1bcfe45e47775b98cce5541f328c4fd46e5eb13d returns an iterator of all blocks in a given priority queue returns an iterator of all blocks in a given priority queue FILE_PATH_CHANGED clear() void clear() for (int i = 0; i < LEVEL; i++) { priorityQueues.get(i).clear(); }
5605 -1009652345apache/hadoopEli Collins1bcfe45e47775b98cce5541f328c4fd46e5eb13d add a block to a under replication queue according to its priority @param block a under replication block @param curReplicas current number of replicas of the block @param expectedReplicas expected number of replicas of the block add a block to a under replication queue according to its priority @param block a under replication block @param curReplicas current number of replicas of the block @param expectedReplicas expected number of replicas of the block FILE_PATH_CHANGED clear() void clear() for (int i = 0; i < LEVEL; i++) { priorityQueues.get(i).clear(); }
5604 -1009652346apache/hadoopEli Collins1bcfe45e47775b98cce5541f328c4fd46e5eb13d Return the priority of a block @param block a under replication block @param curReplicas current number of replicas of the block @param expectedReplicas expected number of replicas of the block Return the priority of a block @param block a under replication block @param curReplicas current number of replicas of the block @param expectedReplicas expected number of replicas of the block FILE_PATH_CHANGED clear() void clear() for (int i = 0; i < LEVEL; i++) { priorityQueues.get(i).clear(); }
5603 -1009652203apache/hadoopEli Collins1bcfe45e47775b98cce5541f328c4fd46e5eb13d Sufficient to rely on super's implementation Sufficient to rely on super's implementation FILE_PATH_CHANGED equals(Object) public boolean equals(Object obj) // Sufficient to rely on super's implementation return (this == obj) || super.equals(obj);
5602 -1009652224apache/hadoopEli Collins1bcfe45e47775b98cce5541f328c4fd46e5eb13d BlockInfo BlockInfoUnderConstruction participates in maps the same way as BlockInfo BlockInfo BlockInfoUnderConstruction participates in maps the same way as BlockInfo FILE_PATH_CHANGED hashCode() public int hashCode() return super.hashCode();
5601 -1009652225apache/hadoopEli Collins1bcfe45e47775b98cce5541f328c4fd46e5eb13d Sufficient to rely on super's implementation Sufficient to rely on super's implementation FILE_PATH_CHANGED equals(Object) public boolean equals(Object obj) // Sufficient to rely on super's implementation return (this == obj) || super.equals(obj);
5600 -1009652349apache/hadoopEli Collins1bcfe45e47775b98cce5541f328c4fd46e5eb13d the block is only in one of the to-do lists if it is in none then data-node already has it the block is only in one of the to-do lists if it is in none then data-node already has it FILE_PATH_CHANGED addBlock(DatanodeDescriptor, Block, String) public void addBlock(DatanodeDescriptor node, Block block, String delHint) throws IOException // decrement number of blocks scheduled to this datanode. node.decBlocksScheduled(); // get the deletion hint node DatanodeDescriptor delHintNode = null; if (delHint != null && delHint.length() != 0) { delHintNode = namesystem.getDatanode(delHint); if (delHintNode == null) { NameNode.stateChangeLog.warn("BLOCK* NameSystem.blockReceived: " + block + " is expected to be removed from an unrecorded node " + delHint); } } // // Modify the blocks->datanode map and node's map. // pendingReplications.remove(block); // blockReceived reports a finalized block Collection toAdd = new LinkedList(); Collection toInvalidate = new LinkedList(); Collection toCorrupt = new LinkedList(); Collection toUC = new LinkedList(); processReportedBlock(node, block, ReplicaState.FINALIZED, toAdd, toInvalidate, toCorrupt, toUC); // the block is only in one of the to-do lists // if it is in none then data-node already has it assert toUC.size() + toAdd.size() + toInvalidate.size() + toCorrupt.size() <= 1 : "The block should be only in one of the lists."; for (StatefulBlockInfo b : toUC) { addStoredBlockUnderConstruction(b.storedBlock, node, b.reportedState); } for (BlockInfo b : toAdd) { addStoredBlock(b, node, delHintNode, true); } for (Block b : toInvalidate) { NameNode.stateChangeLog.info("BLOCK* NameSystem.addBlock: block " + b + " on " + node.getName() + " size " + b.getNumBytes() + " does not belong to any file."); addToInvalidates(b, node); } for (BlockInfo b : toCorrupt) { markBlockAsCorrupt(b, node); }
5599 -1009652350apache/hadoopEli Collins1bcfe45e47775b98cce5541f328c4fd46e5eb13d The next two methods test the various cases under which we must conclude the replica is corrupt, or under construction. These are laid out as switch statements, on the theory that it is easier to understand the combinatorics of reportedState and ucState that way. It should be at least as efficient as boolean expressions. The next two methods test the various cases under which we must conclude the replica is corrupt, or under construction. These are laid out as switch statements, on the theory that it is easier to understand the combinatorics of reportedState and ucState that way. It should be at least as efficient as boolean expressions. FILE_PATH_CHANGED isReplicaCorrupt(Block, ReplicaState, BlockInfo, BlockUCState, DatanodeDescriptor) private boolean isReplicaCorrupt(Block iblk, ReplicaState reportedState, BlockInfo storedBlock, BlockUCState ucState, DatanodeDescriptor dn) switch(reportedState) { case FINALIZED: switch(ucState) { case COMPLETE: case COMMITTED: return (storedBlock.getGenerationStamp() != iblk.getGenerationStamp() || storedBlock.getNumBytes() != iblk.getNumBytes()); default: return false; } case RBW: case RWR: return storedBlock.isComplete(); // should not be reported case RUR: // should not be reported case TEMPORARY: default: FSNamesystem.LOG.warn("Unexpected replica state " + reportedState + " for block: " + storedBlock + " on " + dn.getName() + " size " + storedBlock.getNumBytes()); return true; }
5598 -1009652351apache/hadoopEli Collins1bcfe45e47775b98cce5541f328c4fd46e5eb13d move block to the head of the list move block to the head of the list FILE_PATH_CHANGED reportDiff(DatanodeDescriptor, BlockListAsLongs, Collection, Collection, Collection, Collection, Collection) void reportDiff(DatanodeDescriptor dn, BlockListAsLongs newReport, Collection toAdd, Collection toRemove, Collection toInvalidate, Collection toCorrupt, Collection toUC) // add to under-construction list // place a delimiter in the list which separates blocks // that have been reported from those that have not BlockInfo delimiter = new BlockInfo(new Block(), 1); boolean added = dn.addBlock(delimiter); assert added : "Delimiting block cannot be present in the node"; if (newReport == null) newReport = new BlockListAsLongs(); // scan the report and process newly reported blocks BlockReportIterator itBR = newReport.getBlockReportIterator(); while (itBR.hasNext()) { Block iblk = itBR.next(); ReplicaState iState = itBR.getCurrentReplicaState(); BlockInfo storedBlock = processReportedBlock(dn, iblk, iState, toAdd, toInvalidate, toCorrupt, toUC); // move block to the head of the list if (storedBlock != null && storedBlock.findDatanode(dn) >= 0) dn.moveBlockToHead(storedBlock); } // collect blocks that have not been reported // all of them are next to the delimiter Iterator it = new DatanodeDescriptor.BlockIterator(delimiter.getNext(0), dn); while (it.hasNext()) toRemove.add(it.next()); dn.removeBlock(delimiter);
5597 -1009652352apache/hadoopEli Collins1bcfe45e47775b98cce5541f328c4fd46e5eb13d Initial block reports can be processed a lot more efficiently than ordinary block reports. This shortens NN restart times. Initial block reports can be processed a lot more efficiently than ordinary block reports. This shortens NN restart times. FILE_PATH_CHANGED processReport(DatanodeDescriptor, BlockListAsLongs) public void processReport(DatanodeDescriptor node, BlockListAsLongs report) throws IOException boolean isFirstBlockReport = (node.numBlocks() == 0); if (isFirstBlockReport) { // Initial block reports can be processed a lot more efficiently than // ordinary block reports. This shortens NN restart times. processFirstBlockReport(node, report); return; } // Normal case: // Modify the (block-->datanode) map, according to the difference // between the old and new block report. // Collection toAdd = new LinkedList(); Collection toRemove = new LinkedList(); Collection toInvalidate = new LinkedList(); Collection toCorrupt = new LinkedList(); Collection toUC = new LinkedList(); reportDiff(node, report, toAdd, toRemove, toInvalidate, toCorrupt, toUC); // Process the blocks on each queue for (StatefulBlockInfo b : toUC) { addStoredBlockUnderConstruction(b.storedBlock, node, b.reportedState); } for (Block b : toRemove) { removeStoredBlock(b, node); } for (BlockInfo b : toAdd) { addStoredBlock(b, node, null, true); } for (Block b : toInvalidate) { NameNode.stateChangeLog.info("BLOCK* NameSystem.processReport: block " + b + " on " + node.getName() + " size " + b.getNumBytes() + " does not belong to any file."); addToInvalidates(b, node); } for (BlockInfo b : toCorrupt) { markBlockAsCorrupt(b, node); }
5594 -1009652121apache/hadoopAaron Myers9c83d16c81df608d8fa1075906a7a0b989faef80 None For some reason even with an MBeanException available to them Runtime exceptionscan still find their way through, so treat them the same as MBeanException SATD_ADDED listBeans(JsonGenerator, ObjectName, String, HttpServletResponse) private void listBeans(JsonGenerator jg, ObjectName qry, String attribute, HttpServletResponse response) throws IOException LOG.debug("Listing beans for " + qry); Set names = null; names = mBeanServer.queryNames(qry, null); jg.writeArrayFieldStart("beans"); Iterator it = names.iterator(); while (it.hasNext()) { ObjectName oname = it.next(); MBeanInfo minfo; String code = ""; Object attributeinfo = null; try { minfo = mBeanServer.getMBeanInfo(oname); code = minfo.getClassName(); String prs = ""; try { if ("org.apache.commons.modeler.BaseModelMBean".equals(code)) { prs = "modelerType"; code = (String) mBeanServer.getAttribute(oname, prs); } if (attribute != null) { prs = attribute; attributeinfo = mBeanServer.getAttribute(oname, prs); } } catch (AttributeNotFoundException e) { // If the modelerType attribute was not found, the class name is used // instead. LOG.error("getting attribute " + prs + " of " + oname + " threw an exception", e); } catch (MBeanException e) { // The code inside the attribute getter threw an exception so log it, // and fall back on the class name LOG.error("getting attribute " + prs + " of " + oname + " threw an exception", e); } catch (RuntimeException e) { // For some reason even with an MBeanException available to them // Runtime exceptionscan still find their way through, so treat them // the same as MBeanException LOG.error("getting attribute " + prs + " of " + oname + " threw an exception", e); } catch (ReflectionException e) { // This happens when the code inside the JMX bean (setter?? from the // java docs) threw an exception, so log it and fall back on the // class name LOG.error("getting attribute " + prs + " of " + oname + " threw an exception", e); } } catch (InstanceNotFoundException e) { // Ignored for some reason the bean was not found so don't output it continue; } catch (IntrospectionException e) { // This is an internal error, something odd happened with reflection so // log it and don't output the bean. LOG.error("Problem while trying to process JMX query: " + qry + " with MBean " + oname, e); continue; } catch (ReflectionException e) { // This happens when the code inside the JMX bean threw an exception, so // log it and don't output the bean. LOG.error("Problem while trying to process JMX query: " + qry + " with MBean " + oname, e); continue; } jg.writeStartObject(); jg.writeStringField("name", oname.toString()); jg.writeStringField("modelerType", code); if ((attribute != null) && (attributeinfo == null)) { jg.writeStringField("result", "ERROR"); jg.writeStringField("message", "No attribute with name " + attribute + " was found."); jg.writeEndObject(); jg.writeEndArray(); jg.close(); response.setStatus(HttpServletResponse.SC_NOT_FOUND); return; } if (attribute != null) { writeAttribute(jg, attribute, attributeinfo); } else { MBeanAttributeInfo[] attrs = minfo.getAttributes(); for (int i = 0; i < attrs.length; i++) { writeAttribute(jg, oname, attrs[i]); } } jg.writeEndObject(); } jg.writeEndArray();
5590 -1009652124apache/hadoopEli Collins7e1e4bf50fa83083e762fc267b5215d606a64c3e None Only need todo this once SATD_ADDED compress(byte[], int, int) public synchronized int compress(byte[] b, int off, int len) throws IOException if (b == null) { throw new NullPointerException(); } if (off < 0 || len < 0 || off > b.length - len) { throw new ArrayIndexOutOfBoundsException(); } if (finished || outBuf.position() == 0) { finished = true; return 0; } // Only need todo this once if (compressedBuf.position() == 0) { try { outBuf.limit(outBuf.position()); outBuf.rewind(); int lim = Snappy.compress(outBuf, compressedBuf); compressedBuf.limit(lim); compressedBuf.rewind(); } catch (SnappyException e) { throw new IOException(e); } } int n = (compressedBuf.limit() - compressedBuf.position()) > len ? len : (compressedBuf.limit() - compressedBuf.position()); if (n == 0) { finished = true; return 0; } compressedBuf.get(b, off, n); bytesWritten += n; // Set 'finished' if snappy has consumed all user-data if (compressedBuf.position() == compressedBuf.limit()) { finished = true; outBuf.limit(outBuf.capacity()); outBuf.rewind(); compressedBuf.limit(compressedBuf.capacity()); compressedBuf.rewind(); } return n;
5589 -1009652137apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Hack: ensuring rack name starts with \"/\". SATD_ADDED addChild(Node) public synchronized boolean addChild(Node child) if (!(child instanceof MachineNode)) { throw new IllegalArgumentException("Only MachineNode can be added to RackNode"); } return super.addChild(child);
5588 -1009652138apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Since distributed cache is unmodified in dfs between two job runs, it should not be present more than once in any of the task tracker, in which job ran. SATD_ADDED testDistributedCache() public void testDistributedCache() throws Exception Configuration conf = new Configuration(cluster.getConf()); JTProtocol wovenClient = cluster.getJTClient().getProxy(); // This counter will check for count of a loop, // which might become infinite. int count = 0; // This boolean will decide whether to run job again boolean continueLoop = true; // counter for job Loop int countLoop = 0; // This counter incerases with all the tasktrackers in which tasks ran int taskTrackerCounter = 0; // This will store all the tasktrackers in which tasks ran ArrayList taskTrackerCollection = new ArrayList(); do { SleepJob job = new SleepJob(); job.setConf(conf); Job slpJob = job.createJob(5, 1, 1000, 1000, 100, 100); DistributedCache.createSymlink(conf); URI uri = URI.create(uriPath); DistributedCache.addCacheFile(uri, conf); JobConf jconf = new JobConf(conf); // Controls the job till all verification is done FinishTaskControlAction.configureControlActionForJob(conf); // Submitting the job slpJob.submit(); RunningJob rJob = cluster.getJTClient().getClient().getJob(org.apache.hadoop.mapred.JobID.downgrade(slpJob.getJobID())); // counter for job Loop countLoop++; TTClient tClient = null; JobInfo jInfo = wovenClient.getJobInfo(rJob.getID()); LOG.info("jInfo is :" + jInfo); // Assert if jobInfo is null Assert.assertNotNull("jobInfo is null", jInfo); // Wait for the job to start running. count = 0; while (jInfo.getStatus().getRunState() != JobStatus.RUNNING) { UtilsForTests.waitFor(10000); count++; jInfo = wovenClient.getJobInfo(rJob.getID()); // If the count goes beyond a point, then break; This is to avoid // infinite loop under unforeseen circumstances. Testcase will anyway // fail later. if (count > 10) { Assert.fail("job has not reached running state for more than" + "100 seconds. Failing at this point"); } } LOG.info("job id is :" + rJob.getID().toString()); TaskInfo[] taskInfos = cluster.getJTClient().getProxy().getTaskInfo(rJob.getID()); boolean distCacheFileIsFound; for (TaskInfo taskInfo : taskInfos) { distCacheFileIsFound = false; String[] taskTrackers = taskInfo.getTaskTrackers(); for (String taskTracker : taskTrackers) { // Formatting tasktracker to get just its FQDN taskTracker = UtilsForTests.getFQDNofTT(taskTracker); LOG.info("taskTracker is :" + taskTracker); // This will be entered from the second job onwards if (countLoop > 1) { if (taskTracker != null) { continueLoop = taskTrackerCollection.contains(taskTracker); } if (!continueLoop) { break; } } // Collecting the tasktrackers if (taskTracker != null) taskTrackerCollection.add(taskTracker); // we have loopped through enough number of times to look for task // getting submitted on same tasktrackers.The same tasktracker // for subsequent jobs was not hit maybe because of many number // of tasktrackers. So, testcase has to stop here. if (countLoop > 2) { continueLoop = false; } tClient = cluster.getTTClient(taskTracker); // tClient maybe null because the task is already dead. Ex: setup if (tClient == null) { continue; } String[] localDirs = tClient.getMapredLocalDirs(); int distributedFileCount = 0; // Go to every single path for (String localDir : localDirs) { // Public Distributed cache will always be stored under // mapre.local.dir/tasktracker/archive localDir = localDir + Path.SEPARATOR + TaskTracker.getPublicDistributedCacheDir(); LOG.info("localDir is : " + localDir); // Get file status of all the directories // and files under that path. FileStatus[] fileStatuses = tClient.listStatus(localDir, true, true); for (FileStatus fileStatus : fileStatuses) { Path path = fileStatus.getPath(); LOG.info("path is :" + path.toString()); // Checking if the received path ends with // the distributed filename distCacheFileIsFound = (path.toString()).endsWith(distributedFileName); // If file is found, check for its permission. // Since the file is found break out of loop if (distCacheFileIsFound) { LOG.info("PATH found is :" + path.toString()); distributedFileCount++; String filename = path.getName(); FsPermission fsPerm = fileStatus.getPermission(); Assert.assertTrue("File Permission is not 777", fsPerm.equals(new FsPermission("777"))); } } } // Since distributed cache is unmodified in dfs // between two job runs, it should not be present more than once // in any of the task tracker, in which job ran. if (distributedFileCount > 1) { Assert.fail("The distributed cache file is more than one"); } else if (distributedFileCount < 1) Assert.fail("The distributed cache file is less than one"); if (!distCacheFileIsFound) { Assert.assertEquals("The distributed cache file does not exist", distCacheFileIsFound, false); } } } // Allow the job to continue through MR control job. for (TaskInfo taskInfoRemaining : taskInfos) { FinishTaskControlAction action = new FinishTaskControlAction(TaskID.downgrade(taskInfoRemaining.getTaskID())); Collection tts = cluster.getTTClients(); for (TTClient cli : tts) { cli.getProxy().sendAction(action); } } // Killing the job because all the verification needed // for this testcase is completed. rJob.killJob(); } while (continueLoop);
5587 -1009652139apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None tClient maybe null because the task is already dead. Ex: setup SATD_ADDED testDistributedCache() public void testDistributedCache() throws Exception Configuration conf = new Configuration(cluster.getConf()); JTProtocol wovenClient = cluster.getJTClient().getProxy(); // This counter will check for count of a loop, // which might become infinite. int count = 0; // This boolean will decide whether to run job again boolean continueLoop = true; // counter for job Loop int countLoop = 0; // This counter incerases with all the tasktrackers in which tasks ran int taskTrackerCounter = 0; // This will store all the tasktrackers in which tasks ran ArrayList taskTrackerCollection = new ArrayList(); do { SleepJob job = new SleepJob(); job.setConf(conf); Job slpJob = job.createJob(5, 1, 1000, 1000, 100, 100); DistributedCache.createSymlink(conf); URI uri = URI.create(uriPath); DistributedCache.addCacheFile(uri, conf); JobConf jconf = new JobConf(conf); // Controls the job till all verification is done FinishTaskControlAction.configureControlActionForJob(conf); // Submitting the job slpJob.submit(); RunningJob rJob = cluster.getJTClient().getClient().getJob(org.apache.hadoop.mapred.JobID.downgrade(slpJob.getJobID())); // counter for job Loop countLoop++; TTClient tClient = null; JobInfo jInfo = wovenClient.getJobInfo(rJob.getID()); LOG.info("jInfo is :" + jInfo); // Assert if jobInfo is null Assert.assertNotNull("jobInfo is null", jInfo); // Wait for the job to start running. count = 0; while (jInfo.getStatus().getRunState() != JobStatus.RUNNING) { UtilsForTests.waitFor(10000); count++; jInfo = wovenClient.getJobInfo(rJob.getID()); // If the count goes beyond a point, then break; This is to avoid // infinite loop under unforeseen circumstances. Testcase will anyway // fail later. if (count > 10) { Assert.fail("job has not reached running state for more than" + "100 seconds. Failing at this point"); } } LOG.info("job id is :" + rJob.getID().toString()); TaskInfo[] taskInfos = cluster.getJTClient().getProxy().getTaskInfo(rJob.getID()); boolean distCacheFileIsFound; for (TaskInfo taskInfo : taskInfos) { distCacheFileIsFound = false; String[] taskTrackers = taskInfo.getTaskTrackers(); for (String taskTracker : taskTrackers) { // Formatting tasktracker to get just its FQDN taskTracker = UtilsForTests.getFQDNofTT(taskTracker); LOG.info("taskTracker is :" + taskTracker); // This will be entered from the second job onwards if (countLoop > 1) { if (taskTracker != null) { continueLoop = taskTrackerCollection.contains(taskTracker); } if (!continueLoop) { break; } } // Collecting the tasktrackers if (taskTracker != null) taskTrackerCollection.add(taskTracker); // we have loopped through enough number of times to look for task // getting submitted on same tasktrackers.The same tasktracker // for subsequent jobs was not hit maybe because of many number // of tasktrackers. So, testcase has to stop here. if (countLoop > 2) { continueLoop = false; } tClient = cluster.getTTClient(taskTracker); // tClient maybe null because the task is already dead. Ex: setup if (tClient == null) { continue; } String[] localDirs = tClient.getMapredLocalDirs(); int distributedFileCount = 0; // Go to every single path for (String localDir : localDirs) { // Public Distributed cache will always be stored under // mapre.local.dir/tasktracker/archive localDir = localDir + Path.SEPARATOR + TaskTracker.getPublicDistributedCacheDir(); LOG.info("localDir is : " + localDir); // Get file status of all the directories // and files under that path. FileStatus[] fileStatuses = tClient.listStatus(localDir, true, true); for (FileStatus fileStatus : fileStatuses) { Path path = fileStatus.getPath(); LOG.info("path is :" + path.toString()); // Checking if the received path ends with // the distributed filename distCacheFileIsFound = (path.toString()).endsWith(distributedFileName); // If file is found, check for its permission. // Since the file is found break out of loop if (distCacheFileIsFound) { LOG.info("PATH found is :" + path.toString()); distributedFileCount++; String filename = path.getName(); FsPermission fsPerm = fileStatus.getPermission(); Assert.assertTrue("File Permission is not 777", fsPerm.equals(new FsPermission("777"))); } } } // Since distributed cache is unmodified in dfs // between two job runs, it should not be present more than once // in any of the task tracker, in which job ran. if (distributedFileCount > 1) { Assert.fail("The distributed cache file is more than one"); } else if (distributedFileCount < 1) Assert.fail("The distributed cache file is less than one"); if (!distCacheFileIsFound) { Assert.assertEquals("The distributed cache file does not exist", distCacheFileIsFound, false); } } } // Allow the job to continue through MR control job. for (TaskInfo taskInfoRemaining : taskInfos) { FinishTaskControlAction action = new FinishTaskControlAction(TaskID.downgrade(taskInfoRemaining.getTaskID())); Collection tts = cluster.getTTClients(); for (TTClient cli : tts) { cli.getProxy().sendAction(action); } } // Killing the job because all the verification needed // for this testcase is completed. rJob.killJob(); } while (continueLoop);
5585 -1009652141apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None HDFS ignore the \"x\" bit if the permission. SATD_ADDED clusterSetupAtBegining() public static void clusterSetupAtBegining() throws IOException, LoginException, URISyntaxException Configuration conf = new HdfsConfiguration(); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); fc = FileContext.getFileContext(cluster.getURI(0), conf); defaultWorkingDirectory = fc.makeQualified(new Path("/user/" + UserGroupInformation.getCurrentUser().getShortUserName())); fc.mkdir(defaultWorkingDirectory, FileContext.DEFAULT_PERM, true);
5584 -1009652142apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Shouldn't happen since we do implement Clonable SATD_ADDED clone() public Object clone() try { return super.clone(); } catch (CloneNotSupportedException cnse) { // Shouldn't happen since we do implement Clonable throw new InternalError(cnse.toString()); }
5583 -1009652143apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None TCP_NODELAY is crucial here because of bad interactions between Nagle's Algorithm and Delayed ACKs. With connection keepalive between the client and DN, the conversation looks like: 1. Client -> DN: Read block X 2. DN -> Client: data for block X 3. Client -> DN: Status OK (successful read) 4. Client -> DN: Read block Y The fact that step #3 and #4 are both in the client->DN direction triggers Nagling. If the DN is using delayed ACKs, this results in a delay of 40ms or more. TCP_NODELAY disables nagling and thus avoids this performance disaster. SATD_ADDED addToDeadNodes(DatanodeInfo) void addToDeadNodes(DatanodeInfo dnInfo) deadNodes.put(dnInfo, dnInfo);
5582 -1009652144apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Allow retry since there is no way of knowing whether the cached socket is good until we actually use it. SATD_ADDED addToDeadNodes(DatanodeInfo) void addToDeadNodes(DatanodeInfo dnInfo) deadNodes.put(dnInfo, dnInfo);
5581 -1009652145apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Connect to best DataNode for desired Block, with potential offset only need to get a new access token once SATD_ADDED addToDeadNodes(DatanodeInfo) void addToDeadNodes(DatanodeInfo dnInfo) deadNodes.put(dnInfo, dnInfo);
5580 -1009652146apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Connect to best DataNode for desired Block, with potential offset SATD_ADDED addToDeadNodes(DatanodeInfo) void addToDeadNodes(DatanodeInfo dnInfo) deadNodes.put(dnInfo, dnInfo);
5578 -1009652148apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Finish the map task on the tracker 1. Finishing it here to work around bug in the FakeJobInProgress object SATD_ADDED testReservationOnBlacklistedTracker() public void testReservationOnBlacklistedTracker() throws Exception TaskAttemptID[] taskAttemptID = new TaskAttemptID[3]; JobConf conf = new JobConf(); conf.setSpeculativeExecution(false); conf.setNumMapTasks(2); conf.setNumReduceTasks(2); conf.set(JobContext.REDUCE_FAILURES_MAXPERCENT, ".70"); conf.set(JobContext.MAP_FAILURES_MAX_PERCENT, ".70"); conf.setBoolean(JobContext.SETUP_CLEANUP_NEEDED, false); conf.setMaxTaskFailuresPerTracker(1); FakeJobInProgress job = new FakeJobInProgress(conf, jobTracker); job.setClusterSize(trackers.length); job.initTasks(); TaskTracker tt1 = jobTracker.getTaskTracker(trackers[0]); TaskTracker tt2 = jobTracker.getTaskTracker(trackers[1]); TaskTracker tt3 = jobTracker.getTaskTracker(trackers[2]); TaskTrackerStatus status1 = new TaskTrackerStatus(trackers[0], JobInProgress.convertTrackerNameToHostName(trackers[0]), 0, new ArrayList(), 0, 2, 2); TaskTrackerStatus status2 = new TaskTrackerStatus(trackers[1], JobInProgress.convertTrackerNameToHostName(trackers[1]), 0, new ArrayList(), 0, 2, 2); TaskTrackerStatus status3 = new TaskTrackerStatus(trackers[1], JobInProgress.convertTrackerNameToHostName(trackers[1]), 0, new ArrayList(), 0, 2, 2); tt1.setStatus(status1); tt2.setStatus(status2); tt3.setStatus(status3); tt1.reserveSlots(TaskType.MAP, job, 2); tt1.reserveSlots(TaskType.REDUCE, job, 2); tt3.reserveSlots(TaskType.MAP, job, 2); tt3.reserveSlots(TaskType.REDUCE, job, 2); assertEquals("Trackers not reserved for the job : maps", 2, job.getNumReservedTaskTrackersForMaps()); assertEquals("Trackers not reserved for the job : reduces", 2, job.getNumReservedTaskTrackersForReduces()); ClusterMetrics metrics = jobTracker.getClusterMetrics(); assertEquals("reserved map slots do not match", 4, metrics.getReservedMapSlots()); assertEquals("reserved reduce slots do not match", 4, metrics.getReservedReduceSlots()); /* * FakeJobInProgress.findMapTask does not handle * task failures. So working around it by failing * reduce and blacklisting tracker. * Then finish the map task later. */ TaskAttemptID mTid = job.findMapTask(trackers[0]); TaskAttemptID rTid = job.findReduceTask(trackers[0]); // Task should blacklist the tasktracker. job.failTask(rTid); assertEquals("Tracker 0 not blacklisted for the job", 1, job.getBlackListedTrackers().size()); assertEquals("Extra Trackers reserved for the job : maps", 1, job.getNumReservedTaskTrackersForMaps()); assertEquals("Extra Trackers reserved for the job : reduces", 1, job.getNumReservedTaskTrackersForReduces()); metrics = jobTracker.getClusterMetrics(); assertEquals("reserved map slots do not match", 2, metrics.getReservedMapSlots()); assertEquals("reserved reduce slots do not match", 2, metrics.getReservedReduceSlots()); // Finish the map task on the tracker 1. Finishing it here to work // around bug in the FakeJobInProgress object job.finishTask(mTid); mTid = job.findMapTask(trackers[1]); rTid = job.findReduceTask(trackers[1]); job.finishTask(mTid); job.finishTask(rTid); rTid = job.findReduceTask(trackers[1]); job.finishTask(rTid); assertEquals("Job didnt complete successfully complete", job.getStatus().getRunState(), JobStatus.SUCCEEDED); assertEquals("Trackers not unreserved for the job : maps", 0, job.getNumReservedTaskTrackersForMaps()); assertEquals("Trackers not unreserved for the job : reduces", 0, job.getNumReservedTaskTrackersForReduces()); metrics = jobTracker.getClusterMetrics(); assertEquals("reserved map slots do not match", 0, metrics.getReservedMapSlots()); assertEquals("reserved reduce slots do not match", 0, metrics.getReservedReduceSlots());
5577 -1009652149apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None copy the split and fix up the locations SATD_ADDED getNewFileSplits() public List getNewFileSplits() throws IOException solve(); FileSplit[] result = new FileSplit[realSplits.length]; int left = 0; int right = realSplits.length - 1; for (int i = 0; i < splits.length; ++i) { if (splits[i].isAssigned) { // copy the split and fix up the locations ((TeraFileSplit) realSplits[i]).setLocations(new String[] { splits[i].locations.get(0).hostname }); result[left++] = realSplits[i]; } else { result[right--] = realSplits[i]; } } List ret = new ArrayList(); for (FileSplit fs : result) { ret.add(fs); } return ret;
5576 -1009652150apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None TODO: check for insane values SATD_ADDED readSplitMetaInfo(JobID, FileSystem, Configuration, Path) public static JobSplit.TaskSplitMetaInfo[] readSplitMetaInfo(JobID jobId, FileSystem fs, Configuration conf, Path jobSubmitDir) throws IOException long maxMetaInfoSize = conf.getLong(JTConfig.JT_MAX_JOB_SPLIT_METAINFO_SIZE, 10000000L); Path metaSplitFile = JobSubmissionFiles.getJobSplitMetaFile(jobSubmitDir); FileStatus fStatus = fs.getFileStatus(metaSplitFile); if (maxMetaInfoSize > 0 && fStatus.getLen() > maxMetaInfoSize) { throw new IOException("Split metadata size exceeded " + maxMetaInfoSize + ". Aborting job " + jobId); } FSDataInputStream in = fs.open(metaSplitFile); byte[] header = new byte[JobSplit.META_SPLIT_FILE_HEADER.length]; in.readFully(header); if (!Arrays.equals(JobSplit.META_SPLIT_FILE_HEADER, header)) { throw new IOException("Invalid header on split file"); } int vers = WritableUtils.readVInt(in); if (vers != JobSplit.META_SPLIT_VERSION) { in.close(); throw new IOException("Unsupported split version " + vers); } // TODO: check for insane values int numSplits = WritableUtils.readVInt(in); JobSplit.TaskSplitMetaInfo[] allSplitMetaInfo = new JobSplit.TaskSplitMetaInfo[numSplits]; for (int i = 0; i < numSplits; i++) { JobSplit.SplitMetaInfo splitMetaInfo = new JobSplit.SplitMetaInfo(); splitMetaInfo.readFields(in); JobSplit.TaskSplitIndex splitIndex = new JobSplit.TaskSplitIndex(JobSubmissionFiles.getJobSplitFile(jobSubmitDir).toString(), splitMetaInfo.getStartOffset()); allSplitMetaInfo[i] = new JobSplit.TaskSplitMetaInfo(splitIndex, splitMetaInfo.getLocations(), splitMetaInfo.getInputDataLength()); } in.close(); return allSplitMetaInfo;
5575 -1009652151apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Don't assign reduce tasks to the hilt! Leave some free slots in the cluster for future task-failures, speculative tasks etc. beyond the highest priority job SATD_ADDED assignTasks(TaskTracker) public synchronized List assignTasks(TaskTracker taskTracker) throws IOException TaskTrackerStatus taskTrackerStatus = taskTracker.getStatus(); ClusterStatus clusterStatus = taskTrackerManager.getClusterStatus(); final int numTaskTrackers = clusterStatus.getTaskTrackers(); final int clusterMapCapacity = clusterStatus.getMaxMapTasks(); final int clusterReduceCapacity = clusterStatus.getMaxReduceTasks(); Collection jobQueue = jobQueueJobInProgressListener.getJobQueue(); // // Get map + reduce counts for the current tracker. // final int trackerMapCapacity = taskTrackerStatus.getMaxMapSlots(); final int trackerReduceCapacity = taskTrackerStatus.getMaxReduceSlots(); final int trackerRunningMaps = taskTrackerStatus.countMapTasks(); final int trackerRunningReduces = taskTrackerStatus.countReduceTasks(); // Assigned tasks List assignedTasks = new ArrayList(); // // Compute (running + pending) map and reduce task numbers across pool // int remainingReduceLoad = 0; int remainingMapLoad = 0; synchronized (jobQueue) { for (JobInProgress job : jobQueue) { if (job.getStatus().getRunState() == JobStatus.RUNNING) { remainingMapLoad += (job.desiredMaps() - job.finishedMaps()); if (job.scheduleReduces()) { remainingReduceLoad += (job.desiredReduces() - job.finishedReduces()); } } } } // Compute the 'load factor' for maps and reduces double mapLoadFactor = 0.0; if (clusterMapCapacity > 0) { mapLoadFactor = (double) remainingMapLoad / clusterMapCapacity; } double reduceLoadFactor = 0.0; if (clusterReduceCapacity > 0) { reduceLoadFactor = (double) remainingReduceLoad / clusterReduceCapacity; } // // In the below steps, we allocate first map tasks (if appropriate), // and then reduce tasks if appropriate. We go through all jobs // in order of job arrival; jobs only get serviced if their // predecessors are serviced, too. // // // We assign tasks to the current taskTracker if the given machine // has a workload that's less than the maximum load of that kind of // task. // However, if the cluster is close to getting loaded i.e. we don't // have enough _padding_ for speculative executions etc., we only // schedule the "highest priority" task i.e. the task from the job // with the highest priority. // final int trackerCurrentMapCapacity = Math.min((int) Math.ceil(mapLoadFactor * trackerMapCapacity), trackerMapCapacity); int availableMapSlots = trackerCurrentMapCapacity - trackerRunningMaps; boolean exceededMapPadding = false; if (availableMapSlots > 0) { exceededMapPadding = exceededPadding(true, clusterStatus, trackerMapCapacity); } int numLocalMaps = 0; int numNonLocalMaps = 0; scheduleMaps: for (int i = 0; i < availableMapSlots; ++i) { synchronized (jobQueue) { for (JobInProgress job : jobQueue) { if (job.getStatus().getRunState() != JobStatus.RUNNING) { continue; } Task t = null; // Try to schedule a node-local or rack-local Map task t = job.obtainNewLocalMapTask(taskTrackerStatus, numTaskTrackers, taskTrackerManager.getNumberOfUniqueHosts()); if (t != null) { assignedTasks.add(t); ++numLocalMaps; // Don't assign map tasks to the hilt! // Leave some free slots in the cluster for future task-failures, // speculative tasks etc. beyond the highest priority job if (exceededMapPadding) { break scheduleMaps; } // Try all jobs again for the next Map task break; } // Try to schedule a node-local or rack-local Map task t = job.obtainNewNonLocalMapTask(taskTrackerStatus, numTaskTrackers, taskTrackerManager.getNumberOfUniqueHosts()); if (t != null) { assignedTasks.add(t); ++numNonLocalMaps; // We assign at most 1 off-switch or speculative task // This is to prevent TaskTrackers from stealing local-tasks // from other TaskTrackers. break scheduleMaps; } } } } int assignedMaps = assignedTasks.size(); // // Same thing, but for reduce tasks // However we _never_ assign more than 1 reduce task per heartbeat // final int trackerCurrentReduceCapacity = Math.min((int) Math.ceil(reduceLoadFactor * trackerReduceCapacity), trackerReduceCapacity); final int availableReduceSlots = Math.min((trackerCurrentReduceCapacity - trackerRunningReduces), 1); boolean exceededReducePadding = false; if (availableReduceSlots > 0) { exceededReducePadding = exceededPadding(false, clusterStatus, trackerReduceCapacity); synchronized (jobQueue) { for (JobInProgress job : jobQueue) { if (job.getStatus().getRunState() != JobStatus.RUNNING || job.numReduceTasks == 0) { continue; } Task t = job.obtainNewReduceTask(taskTrackerStatus, numTaskTrackers, taskTrackerManager.getNumberOfUniqueHosts()); if (t != null) { assignedTasks.add(t); break; } // Don't assign reduce tasks to the hilt! // Leave some free slots in the cluster for future task-failures, // speculative tasks etc. beyond the highest priority job if (exceededReducePadding) { break; } } } } if (LOG.isDebugEnabled()) { LOG.debug("Task assignments for " + taskTrackerStatus.getTrackerName() + " --> " + "[" + mapLoadFactor + ", " + trackerMapCapacity + ", " + trackerCurrentMapCapacity + ", " + trackerRunningMaps + "] -> [" + (trackerCurrentMapCapacity - trackerRunningMaps) + ", " + assignedMaps + " (" + numLocalMaps + ", " + numNonLocalMaps + ")] [" + reduceLoadFactor + ", " + trackerReduceCapacity + ", " + trackerCurrentReduceCapacity + "," + trackerRunningReduces + "] -> [" + (trackerCurrentReduceCapacity - trackerRunningReduces) + ", " + (assignedTasks.size() - assignedMaps) + "]"); } return assignedTasks;
5574 -1009652152apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Don't assign map tasks to the hilt! Leave some free slots in the cluster for future task-failures, speculative tasks etc. beyond the highest priority job SATD_ADDED assignTasks(TaskTracker) public synchronized List assignTasks(TaskTracker taskTracker) throws IOException TaskTrackerStatus taskTrackerStatus = taskTracker.getStatus(); ClusterStatus clusterStatus = taskTrackerManager.getClusterStatus(); final int numTaskTrackers = clusterStatus.getTaskTrackers(); final int clusterMapCapacity = clusterStatus.getMaxMapTasks(); final int clusterReduceCapacity = clusterStatus.getMaxReduceTasks(); Collection jobQueue = jobQueueJobInProgressListener.getJobQueue(); // // Get map + reduce counts for the current tracker. // final int trackerMapCapacity = taskTrackerStatus.getMaxMapSlots(); final int trackerReduceCapacity = taskTrackerStatus.getMaxReduceSlots(); final int trackerRunningMaps = taskTrackerStatus.countMapTasks(); final int trackerRunningReduces = taskTrackerStatus.countReduceTasks(); // Assigned tasks List assignedTasks = new ArrayList(); // // Compute (running + pending) map and reduce task numbers across pool // int remainingReduceLoad = 0; int remainingMapLoad = 0; synchronized (jobQueue) { for (JobInProgress job : jobQueue) { if (job.getStatus().getRunState() == JobStatus.RUNNING) { remainingMapLoad += (job.desiredMaps() - job.finishedMaps()); if (job.scheduleReduces()) { remainingReduceLoad += (job.desiredReduces() - job.finishedReduces()); } } } } // Compute the 'load factor' for maps and reduces double mapLoadFactor = 0.0; if (clusterMapCapacity > 0) { mapLoadFactor = (double) remainingMapLoad / clusterMapCapacity; } double reduceLoadFactor = 0.0; if (clusterReduceCapacity > 0) { reduceLoadFactor = (double) remainingReduceLoad / clusterReduceCapacity; } // // In the below steps, we allocate first map tasks (if appropriate), // and then reduce tasks if appropriate. We go through all jobs // in order of job arrival; jobs only get serviced if their // predecessors are serviced, too. // // // We assign tasks to the current taskTracker if the given machine // has a workload that's less than the maximum load of that kind of // task. // However, if the cluster is close to getting loaded i.e. we don't // have enough _padding_ for speculative executions etc., we only // schedule the "highest priority" task i.e. the task from the job // with the highest priority. // final int trackerCurrentMapCapacity = Math.min((int) Math.ceil(mapLoadFactor * trackerMapCapacity), trackerMapCapacity); int availableMapSlots = trackerCurrentMapCapacity - trackerRunningMaps; boolean exceededMapPadding = false; if (availableMapSlots > 0) { exceededMapPadding = exceededPadding(true, clusterStatus, trackerMapCapacity); } int numLocalMaps = 0; int numNonLocalMaps = 0; scheduleMaps: for (int i = 0; i < availableMapSlots; ++i) { synchronized (jobQueue) { for (JobInProgress job : jobQueue) { if (job.getStatus().getRunState() != JobStatus.RUNNING) { continue; } Task t = null; // Try to schedule a node-local or rack-local Map task t = job.obtainNewLocalMapTask(taskTrackerStatus, numTaskTrackers, taskTrackerManager.getNumberOfUniqueHosts()); if (t != null) { assignedTasks.add(t); ++numLocalMaps; // Don't assign map tasks to the hilt! // Leave some free slots in the cluster for future task-failures, // speculative tasks etc. beyond the highest priority job if (exceededMapPadding) { break scheduleMaps; } // Try all jobs again for the next Map task break; } // Try to schedule a node-local or rack-local Map task t = job.obtainNewNonLocalMapTask(taskTrackerStatus, numTaskTrackers, taskTrackerManager.getNumberOfUniqueHosts()); if (t != null) { assignedTasks.add(t); ++numNonLocalMaps; // We assign at most 1 off-switch or speculative task // This is to prevent TaskTrackers from stealing local-tasks // from other TaskTrackers. break scheduleMaps; } } } } int assignedMaps = assignedTasks.size(); // // Same thing, but for reduce tasks // However we _never_ assign more than 1 reduce task per heartbeat // final int trackerCurrentReduceCapacity = Math.min((int) Math.ceil(reduceLoadFactor * trackerReduceCapacity), trackerReduceCapacity); final int availableReduceSlots = Math.min((trackerCurrentReduceCapacity - trackerRunningReduces), 1); boolean exceededReducePadding = false; if (availableReduceSlots > 0) { exceededReducePadding = exceededPadding(false, clusterStatus, trackerReduceCapacity); synchronized (jobQueue) { for (JobInProgress job : jobQueue) { if (job.getStatus().getRunState() != JobStatus.RUNNING || job.numReduceTasks == 0) { continue; } Task t = job.obtainNewReduceTask(taskTrackerStatus, numTaskTrackers, taskTrackerManager.getNumberOfUniqueHosts()); if (t != null) { assignedTasks.add(t); break; } // Don't assign reduce tasks to the hilt! // Leave some free slots in the cluster for future task-failures, // speculative tasks etc. beyond the highest priority job if (exceededReducePadding) { break; } } } } if (LOG.isDebugEnabled()) { LOG.debug("Task assignments for " + taskTrackerStatus.getTrackerName() + " --> " + "[" + mapLoadFactor + ", " + trackerMapCapacity + ", " + trackerCurrentMapCapacity + ", " + trackerRunningMaps + "] -> [" + (trackerCurrentMapCapacity - trackerRunningMaps) + ", " + assignedMaps + " (" + numLocalMaps + ", " + numNonLocalMaps + ")] [" + reduceLoadFactor + ", " + trackerReduceCapacity + ", " + trackerCurrentReduceCapacity + "," + trackerRunningReduces + "] -> [" + (trackerCurrentReduceCapacity - trackerRunningReduces) + ", " + (assignedTasks.size() - assignedMaps) + "]"); } return assignedTasks;
5573 -1009652153apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None no more retries... Give up SATD_ADDED parseLong(String[], int) private long parseLong(String[] args, int offset) if (offset == args.length) { throw new IllegalArgumentException(" not specified in " + cmd); } long n = StringUtils.TraditionalBinaryPrefix.string2long(args[offset]); if (n <= 0) { throw new IllegalArgumentException("n = " + n + " <= 0 in " + cmd); } return n;
5572 -1009652154apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None TODO: when modification times can be set, directories should be emitted to reducers so they might be preserved. Also, mkdirs does SATD_ADDED parseLong(String[], int) private long parseLong(String[] args, int offset) if (offset == args.length) { throw new IllegalArgumentException(" not specified in " + cmd); } long n = StringUtils.TraditionalBinaryPrefix.string2long(args[offset]); if (n <= 0) { throw new IllegalArgumentException("n = " + n + " <= 0 in " + cmd); } return n;
5571 -1009652155apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None work around 256-byte/22-splits issue SATD_ADDED doSingleBzip2BufferSize(JobConf) private static void doSingleBzip2BufferSize(JobConf jConf) throws IOException TextInputFormat format = new TextInputFormat(); format.configure(jConf); // work around 256-byte/22-splits issue format.setMinSplitSize(5500); // here's Nth pair of DecompressorStreams: InputSplit[] splits = format.getSplits(jConf, 100); assertEquals("compressed splits == 2", 2, splits.length); FileSplit tmp = (FileSplit) splits[0]; if (tmp.getPath().getName().equals("testCompressThenConcat.txt.gz")) { System.out.println(" (swapping)"); splits[0] = splits[1]; splits[1] = tmp; } // testConcatThenCompress (single) List results = readSplit(format, splits[0], jConf); assertEquals("splits[0] length (num lines)", 84, results.size()); assertEquals("splits[0][0]", "Call me Ishmael. Some years ago--never mind how long precisely--having", results.get(0).toString()); assertEquals("splits[0][42]", "Tell me, does the magnetic virtue of the needles of the compasses of", results.get(42).toString()); // testCompressThenConcat (multi) results = readSplit(format, splits[1], jConf); assertEquals("splits[1] length (num lines)", 84, results.size()); assertEquals("splits[1][0]", "Call me Ishmael. Some years ago--never mind how long precisely--having", results.get(0).toString()); assertEquals("splits[1][42]", "Tell me, does the magnetic virtue of the needles of the compasses of", results.get(42).toString());
5570 -1009652156apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None ideally would add some offsets/shifts in here (e.g., via extra header data?), but...significant work to hand-generate each header, and no bzip2 spec for reference SATD_ADDED doMultipleBzip2BufferSizes(JobConf, boolean) private static void doMultipleBzip2BufferSizes(JobConf jConf, boolean useNative) throws IOException System.out.println(COLOR_MAGENTA + "doMultipleBzip2BufferSizes() using " + "default bzip2 decompressor" + COLOR_NORMAL); jConf.setBoolean("io.native.lib.available", useNative); int bufferSize; // ideally would add some offsets/shifts in here (e.g., via extra header // data?), but...significant work to hand-generate each header, and no // bzip2 spec for reference for (bufferSize = 1; bufferSize < 34; ++bufferSize) { jConf.setInt("io.file.buffer.size", bufferSize); doSingleBzip2BufferSize(jConf); } bufferSize = 512; jConf.setInt("io.file.buffer.size", bufferSize); doSingleBzip2BufferSize(jConf); bufferSize = 1024; jConf.setInt("io.file.buffer.size", bufferSize); doSingleBzip2BufferSize(jConf); bufferSize = 2 * 1024; jConf.setInt("io.file.buffer.size", bufferSize); doSingleBzip2BufferSize(jConf); bufferSize = 4 * 1024; jConf.setInt("io.file.buffer.size", bufferSize); doSingleBzip2BufferSize(jConf); bufferSize = 63 * 1024; jConf.setInt("io.file.buffer.size", bufferSize); doSingleBzip2BufferSize(jConf); bufferSize = 64 * 1024; jConf.setInt("io.file.buffer.size", bufferSize); doSingleBzip2BufferSize(jConf); bufferSize = 65 * 1024; jConf.setInt("io.file.buffer.size", bufferSize); doSingleBzip2BufferSize(jConf); bufferSize = 127 * 1024; jConf.setInt("io.file.buffer.size", bufferSize); doSingleBzip2BufferSize(jConf); bufferSize = 128 * 1024; jConf.setInt("io.file.buffer.size", bufferSize); doSingleBzip2BufferSize(jConf); bufferSize = 129 * 1024; jConf.setInt("io.file.buffer.size", bufferSize); doSingleBzip2BufferSize(jConf);
5569 -1009652157apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None work around 2-byte splits issue [135 splits for a 208-byte file and a 62-byte file(!)] SATD_ADDED testBzip2() public void testBzip2() throws IOException JobConf jobConf = new JobConf(defaultConf); CompressionCodec bzip2 = new BZip2Codec(); ReflectionUtils.setConf(bzip2, jobConf); localFs.delete(workDir, true); System.out.println(COLOR_BR_CYAN + "testBzip2() using non-native CBZip2InputStream (presumably)" + COLOR_NORMAL); // copy prebuilt (correct!) version of concat.bz2 to HDFS final String fn = "concat" + bzip2.getDefaultExtension(); Path fnLocal = new Path(System.getProperty("test.concat.data", "/tmp"), fn); Path fnHDFS = new Path(workDir, fn); localFs.copyFromLocalFile(fnLocal, fnHDFS); writeFile(localFs, new Path(workDir, "part2.txt.bz2"), bzip2, "this is a test\nof bzip2\n"); FileInputFormat.setInputPaths(jobConf, workDir); // extends FileInputFormat TextInputFormat format = new TextInputFormat(); format.configure(jobConf); // work around 2-byte splits issue format.setMinSplitSize(256); // [135 splits for a 208-byte file and a 62-byte file(!)] InputSplit[] splits = format.getSplits(jobConf, 100); assertEquals("compressed splits == 2", 2, splits.length); FileSplit tmp = (FileSplit) splits[0]; if (tmp.getPath().getName().equals("part2.txt.bz2")) { splits[0] = splits[1]; splits[1] = tmp; } List results = readSplit(format, splits[0], jobConf); assertEquals("splits[0] num lines", 6, results.size()); assertEquals("splits[0][5]", "member #3", results.get(5).toString()); results = readSplit(format, splits[1], jobConf); assertEquals("splits[1] num lines", 2, results.size()); assertEquals("splits[1][0]", "this is a test", results.get(0).toString()); assertEquals("splits[1][1]", "of bzip2", results.get(1).toString());
5568 -1009652158apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None ideally would add some offsets/shifts in here (e.g., via extra fields of various sizes), but...significant work to hand-generate each header SATD_ADDED doMultipleGzipBufferSizes(JobConf, boolean) private static void doMultipleGzipBufferSizes(JobConf jConf, boolean useNative) throws IOException System.out.println(COLOR_YELLOW + "doMultipleGzipBufferSizes() using " + (useNative ? "GzipZlibDecompressor" : "BuiltInGzipDecompressor") + COLOR_NORMAL); jConf.setBoolean("io.native.lib.available", useNative); int bufferSize; // ideally would add some offsets/shifts in here (e.g., via extra fields // of various sizes), but...significant work to hand-generate each header for (bufferSize = 1; bufferSize < 34; ++bufferSize) { jConf.setInt("io.file.buffer.size", bufferSize); doSingleGzipBufferSize(jConf); } bufferSize = 512; jConf.setInt("io.file.buffer.size", bufferSize); doSingleGzipBufferSize(jConf); bufferSize = 1024; jConf.setInt("io.file.buffer.size", bufferSize); doSingleGzipBufferSize(jConf); bufferSize = 2 * 1024; jConf.setInt("io.file.buffer.size", bufferSize); doSingleGzipBufferSize(jConf); bufferSize = 4 * 1024; jConf.setInt("io.file.buffer.size", bufferSize); doSingleGzipBufferSize(jConf); bufferSize = 63 * 1024; jConf.setInt("io.file.buffer.size", bufferSize); doSingleGzipBufferSize(jConf); bufferSize = 64 * 1024; jConf.setInt("io.file.buffer.size", bufferSize); doSingleGzipBufferSize(jConf); bufferSize = 65 * 1024; jConf.setInt("io.file.buffer.size", bufferSize); doSingleGzipBufferSize(jConf); bufferSize = 127 * 1024; jConf.setInt("io.file.buffer.size", bufferSize); doSingleGzipBufferSize(jConf); bufferSize = 128 * 1024; jConf.setInt("io.file.buffer.size", bufferSize); doSingleGzipBufferSize(jConf); bufferSize = 129 * 1024; jConf.setInt("io.file.buffer.size", bufferSize); doSingleGzipBufferSize(jConf);
5567 -1009652159apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None test GzipZlibDecompressor (native), just to be sure (FIXME? could move this call to testGzip(), but would need filename setup above) (alternatively, maybe just nuke testGzip() and extend this?) SATD_ADDED testBuiltInGzipDecompressor() public void testBuiltInGzipDecompressor() throws IOException JobConf jobConf = new JobConf(defaultConf); jobConf.setBoolean("io.native.lib.available", false); CompressionCodec gzip = new GzipCodec(); ReflectionUtils.setConf(gzip, jobConf); localFs.delete(workDir, true); assertEquals("[non-native (Java) codec]", org.apache.hadoop.io.compress.zlib.BuiltInGzipDecompressor.class, gzip.getDecompressorType()); System.out.println(COLOR_BR_YELLOW + "testBuiltInGzipDecompressor() using" + " non-native (Java Inflater) Decompressor (" + gzip.getDecompressorType() + ")" + COLOR_NORMAL); // copy single-member test file to HDFS String fn1 = "testConcatThenCompress.txt" + gzip.getDefaultExtension(); Path fnLocal1 = new Path(System.getProperty("test.concat.data", "/tmp"), fn1); Path fnHDFS1 = new Path(workDir, fn1); localFs.copyFromLocalFile(fnLocal1, fnHDFS1); // copy multiple-member test file to HDFS // (actually in "seekable gzip" format, a la JIRA PIG-42) String fn2 = "testCompressThenConcat.txt" + gzip.getDefaultExtension(); Path fnLocal2 = new Path(System.getProperty("test.concat.data", "/tmp"), fn2); Path fnHDFS2 = new Path(workDir, fn2); localFs.copyFromLocalFile(fnLocal2, fnHDFS2); FileInputFormat.setInputPaths(jobConf, workDir); // here's first pair of DecompressorStreams: final FileInputStream in1 = new FileInputStream(fnLocal1.toString()); final FileInputStream in2 = new FileInputStream(fnLocal2.toString()); assertEquals("concat bytes available", 2734, in1.available()); // w/hdr CRC assertEquals("concat bytes available", 3413, in2.available()); CompressionInputStream cin2 = gzip.createInputStream(in2); LineReader in = new LineReader(cin2); Text out = new Text(); int numBytes, totalBytes = 0, lineNum = 0; while ((numBytes = in.readLine(out)) > 0) { ++lineNum; totalBytes += numBytes; } in.close(); assertEquals("total uncompressed bytes in concatenated test file", 5346, totalBytes); assertEquals("total uncompressed lines in concatenated test file", 84, lineNum); // test BuiltInGzipDecompressor with lots of different input-buffer sizes doMultipleGzipBufferSizes(jobConf, false); // test GzipZlibDecompressor (native), just to be sure // (FIXME? could move this call to testGzip(), but would need filename // setup above) (alternatively, maybe just nuke testGzip() and extend this?) doMultipleGzipBufferSizes(jobConf, true);
5566 -1009652160apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Wrap FSDataInputStream for efficient backward seeks Keep track on position with respect encapsulated FSDataInputStream SATD_ADDED init() public void init() throws IOException LOG.info("StreamBaseRecordReader.init: " + " start_=" + start_ + " end_=" + end_ + " length_=" + length_ + " start_ > in_.getPos() =" + (start_ > in_.getPos()) + " " + start_ + " > " + in_.getPos()); if (start_ > in_.getPos()) { in_.seek(start_); } pos_ = start_; bin_ = new BufferedInputStream(in_); seekNextRecordBoundary();
5565 -1009652161apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None TODO Write resources version no too long #1 long #2 long #3 long #4 SATD_ADDED write(DataOutput) public void write(DataOutput out) throws IOException // TODO Write resources version no too // long #1 WritableUtils.writeVLong(out, cumulativeCpuUsage); // long #2 WritableUtils.writeVLong(out, virtualMemoryUsage); // long #3 WritableUtils.writeVLong(out, physicalMemoryUsage); // long #4 WritableUtils.writeVLong(out, heapUsage);
5563 -1009652163apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Do a small regular read. Very likely this will leave unread data on the socket and make the socket uncacheable. SATD_ADDED run() public void run() for (int i = 0; i < N_ITERATIONS; ++i) { int startOff = rand.nextInt((int) fileSize); int len = 0; try { double p = rand.nextDouble(); if (p < PROPORTION_NON_POSITIONAL_READ) { // Do a small regular read. Very likely this will leave unread // data on the socket and make the socket uncacheable. len = Math.min(rand.nextInt(64), (int) fileSize - startOff); read(startOff, len); bytesRead += len; } else { // Do a positional read most of the time. len = rand.nextInt((int) (fileSize - startOff)); pRead(startOff, len); bytesRead += len; } } catch (Exception ex) { LOG.error(getName() + ": Error while testing read at " + startOff + " length " + len); error = true; fail(ex.getMessage()); } }
5562 -1009652164apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Move the previous metafile into the current one. SATD_ADDED thistest(Configuration, DFSTestUtil) private void thistest(Configuration conf, DFSTestUtil util) throws Exception MiniDFSCluster cluster = null; int numDataNodes = 2; short replFactor = 2; Random random = new Random(); try { cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDataNodes).build(); cluster.waitActive(); FileSystem fs = cluster.getFileSystem(); util.createFiles(fs, "/srcdat", replFactor); util.waitReplication(fs, "/srcdat", (short) 2); // Now deliberately remove/truncate meta blocks from the first // directory of the first datanode. The complete absense of a meta // file disallows this Datanode to send data to another datanode. // However, a client is alowed access to this block. // File storageDir = MiniDFSCluster.getStorageDir(0, 1); String bpid = cluster.getNamesystem().getBlockPoolId(); File data_dir = MiniDFSCluster.getFinalizedDir(storageDir, bpid); assertTrue("data directory does not exist", data_dir.exists()); File[] blocks = data_dir.listFiles(); assertTrue("Blocks do not exist in data-dir", (blocks != null) && (blocks.length > 0)); int num = 0; for (int idx = 0; idx < blocks.length; idx++) { if (blocks[idx].getName().startsWith("blk_") && blocks[idx].getName().endsWith(".meta")) { num++; if (num % 3 == 0) { // // remove .meta file // System.out.println("Deliberately removing file " + blocks[idx].getName()); assertTrue("Cannot remove file.", blocks[idx].delete()); } else if (num % 3 == 1) { // // shorten .meta file // RandomAccessFile file = new RandomAccessFile(blocks[idx], "rw"); FileChannel channel = file.getChannel(); int newsize = random.nextInt((int) channel.size() / 2); System.out.println("Deliberately truncating file " + blocks[idx].getName() + " to size " + newsize + " bytes."); channel.truncate(newsize); file.close(); } else { // // corrupt a few bytes of the metafile // RandomAccessFile file = new RandomAccessFile(blocks[idx], "rw"); FileChannel channel = file.getChannel(); long position = 0; // // The very first time, corrupt the meta header at offset 0 // if (num != 2) { position = (long) random.nextInt((int) channel.size()); } int length = random.nextInt((int) (channel.size() - position + 1)); byte[] buffer = new byte[length]; random.nextBytes(buffer); channel.write(ByteBuffer.wrap(buffer), position); System.out.println("Deliberately corrupting file " + blocks[idx].getName() + " at offset " + position + " length " + length); file.close(); } } } // // Now deliberately corrupt all meta blocks from the second // directory of the first datanode // storageDir = MiniDFSCluster.getStorageDir(0, 1); data_dir = MiniDFSCluster.getFinalizedDir(storageDir, bpid); assertTrue("data directory does not exist", data_dir.exists()); blocks = data_dir.listFiles(); assertTrue("Blocks do not exist in data-dir", (blocks != null) && (blocks.length > 0)); int count = 0; File previous = null; for (int idx = 0; idx < blocks.length; idx++) { if (blocks[idx].getName().startsWith("blk_") && blocks[idx].getName().endsWith(".meta")) { // // Move the previous metafile into the current one. // count++; if (count % 2 == 0) { System.out.println("Deliberately insertimg bad crc into files " + blocks[idx].getName() + " " + previous.getName()); assertTrue("Cannot remove file.", blocks[idx].delete()); assertTrue("Cannot corrupt meta file.", previous.renameTo(blocks[idx])); assertTrue("Cannot recreate empty meta file.", previous.createNewFile()); previous = null; } else { previous = blocks[idx]; } } } // // Only one replica is possibly corrupted. The other replica should still // be good. Verify. // assertTrue("Corrupted replicas not handled properly.", util.checkFiles(fs, "/srcdat")); System.out.println("All File still have a valid replica"); // // set replication factor back to 1. This causes only one replica of // of each block to remain in HDFS. The check is to make sure that // the corrupted replica generated above is the one that gets deleted. // This test is currently disabled until HADOOP-1557 is solved. // util.setReplication(fs, "/srcdat", (short) 1); // util.waitReplication(fs, "/srcdat", (short)1); // System.out.println("All Files done with removing replicas"); // assertTrue("Excess replicas deleted. Corrupted replicas found.", // util.checkFiles(fs, "/srcdat")); System.out.println("The excess-corrupted-replica test is disabled " + " pending HADOOP-1557"); util.cleanup(fs, "/srcdat"); } finally { if (cluster != null) { cluster.shutdown(); } }
5561 -1009652165apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None reduce task has a fixed split of progress amongst copy, shuffle and reduce phases. SATD_ADDED setup() public static void setup() throws Exception fs = FileSystem.getLocal(new Configuration()); fs.delete(testRootTempDir, true); fs.mkdirs(testRootTempDir);
5560 -1009652166apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None verify policy deletes the correct blocks. companion blocks should be evenly distributed. SATD_ADDED testDeleteReplica() public void testDeleteReplica() throws IOException setupCluster(); try { // Set the policy to default policy to place the block in the default way setBlockPlacementPolicy(namesystem, new BlockPlacementPolicyDefault(conf, namesystem, namesystem.clusterMap)); DatanodeDescriptor datanode1 = namesystem.datanodeMap.values().iterator().next(); String source = "/dir/file"; String parity = xorPrefix + source; final Path parityPath = new Path(parity); DFSTestUtil.createFile(fs, parityPath, 3, (short) 1, 0L); DFSTestUtil.waitReplication(fs, parityPath, (short) 1); // start one more datanode cluster.startDataNodes(conf, 1, true, null, rack2, host2, null); DatanodeDescriptor datanode2 = null; for (DatanodeDescriptor d : namesystem.datanodeMap.values()) { if (!d.getName().equals(datanode1.getName())) { datanode2 = d; } } Assert.assertTrue(datanode2 != null); cluster.waitActive(); final Path sourcePath = new Path(source); DFSTestUtil.createFile(fs, sourcePath, 5, (short) 2, 0L); DFSTestUtil.waitReplication(fs, sourcePath, (short) 2); refreshPolicy(); Assert.assertEquals(parity, policy.getParityFile(source)); Assert.assertEquals(source, policy.getSourceFile(parity, xorPrefix)); List sourceBlocks = getBlocks(namesystem, source); List parityBlocks = getBlocks(namesystem, parity); Assert.assertEquals(5, sourceBlocks.size()); Assert.assertEquals(3, parityBlocks.size()); // verify the result of getCompanionBlocks() Collection companionBlocks; companionBlocks = getCompanionBlocks(namesystem, policy, sourceBlocks.get(0).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 0, 1 }, new int[] { 0 }); companionBlocks = getCompanionBlocks(namesystem, policy, sourceBlocks.get(1).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 0, 1 }, new int[] { 0 }); companionBlocks = getCompanionBlocks(namesystem, policy, sourceBlocks.get(2).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 2, 3 }, new int[] { 1 }); companionBlocks = getCompanionBlocks(namesystem, policy, sourceBlocks.get(3).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 2, 3 }, new int[] { 1 }); companionBlocks = getCompanionBlocks(namesystem, policy, sourceBlocks.get(4).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 4 }, new int[] { 2 }); companionBlocks = getCompanionBlocks(namesystem, policy, parityBlocks.get(0).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 0, 1 }, new int[] { 0 }); companionBlocks = getCompanionBlocks(namesystem, policy, parityBlocks.get(1).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 2, 3 }, new int[] { 1 }); companionBlocks = getCompanionBlocks(namesystem, policy, parityBlocks.get(2).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 4 }, new int[] { 2 }); // Set the policy back to raid policy. We have to create a new object // here to clear the block location cache refreshPolicy(); setBlockPlacementPolicy(namesystem, policy); // verify policy deletes the correct blocks. companion blocks should be // evenly distributed. fs.setReplication(sourcePath, (short) 1); DFSTestUtil.waitReplication(fs, sourcePath, (short) 1); Map counters = new HashMap(); refreshPolicy(); for (int i = 0; i < parityBlocks.size(); i++) { companionBlocks = getCompanionBlocks(namesystem, policy, parityBlocks.get(i).getBlock()); counters = BlockPlacementPolicyRaid.countCompanionBlocks(companionBlocks, false); Assert.assertTrue(counters.get(datanode1.getName()) >= 1 && counters.get(datanode1.getName()) <= 2); Assert.assertTrue(counters.get(datanode1.getName()) + counters.get(datanode2.getName()) == companionBlocks.size()); counters = BlockPlacementPolicyRaid.countCompanionBlocks(companionBlocks, true); Assert.assertTrue(counters.get(datanode1.getParent().getName()) >= 1 && counters.get(datanode1.getParent().getName()) <= 2); Assert.assertTrue(counters.get(datanode1.getParent().getName()) + counters.get(datanode2.getParent().getName()) == companionBlocks.size()); } } finally { if (cluster != null) { cluster.shutdown(); } }
5559 -1009652167apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Set the policy to default policy to place the block in the default way SATD_ADDED testDeleteReplica() public void testDeleteReplica() throws IOException setupCluster(); try { // Set the policy to default policy to place the block in the default way setBlockPlacementPolicy(namesystem, new BlockPlacementPolicyDefault(conf, namesystem, namesystem.clusterMap)); DatanodeDescriptor datanode1 = namesystem.datanodeMap.values().iterator().next(); String source = "/dir/file"; String parity = xorPrefix + source; final Path parityPath = new Path(parity); DFSTestUtil.createFile(fs, parityPath, 3, (short) 1, 0L); DFSTestUtil.waitReplication(fs, parityPath, (short) 1); // start one more datanode cluster.startDataNodes(conf, 1, true, null, rack2, host2, null); DatanodeDescriptor datanode2 = null; for (DatanodeDescriptor d : namesystem.datanodeMap.values()) { if (!d.getName().equals(datanode1.getName())) { datanode2 = d; } } Assert.assertTrue(datanode2 != null); cluster.waitActive(); final Path sourcePath = new Path(source); DFSTestUtil.createFile(fs, sourcePath, 5, (short) 2, 0L); DFSTestUtil.waitReplication(fs, sourcePath, (short) 2); refreshPolicy(); Assert.assertEquals(parity, policy.getParityFile(source)); Assert.assertEquals(source, policy.getSourceFile(parity, xorPrefix)); List sourceBlocks = getBlocks(namesystem, source); List parityBlocks = getBlocks(namesystem, parity); Assert.assertEquals(5, sourceBlocks.size()); Assert.assertEquals(3, parityBlocks.size()); // verify the result of getCompanionBlocks() Collection companionBlocks; companionBlocks = getCompanionBlocks(namesystem, policy, sourceBlocks.get(0).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 0, 1 }, new int[] { 0 }); companionBlocks = getCompanionBlocks(namesystem, policy, sourceBlocks.get(1).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 0, 1 }, new int[] { 0 }); companionBlocks = getCompanionBlocks(namesystem, policy, sourceBlocks.get(2).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 2, 3 }, new int[] { 1 }); companionBlocks = getCompanionBlocks(namesystem, policy, sourceBlocks.get(3).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 2, 3 }, new int[] { 1 }); companionBlocks = getCompanionBlocks(namesystem, policy, sourceBlocks.get(4).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 4 }, new int[] { 2 }); companionBlocks = getCompanionBlocks(namesystem, policy, parityBlocks.get(0).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 0, 1 }, new int[] { 0 }); companionBlocks = getCompanionBlocks(namesystem, policy, parityBlocks.get(1).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 2, 3 }, new int[] { 1 }); companionBlocks = getCompanionBlocks(namesystem, policy, parityBlocks.get(2).getBlock()); verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, new int[] { 4 }, new int[] { 2 }); // Set the policy back to raid policy. We have to create a new object // here to clear the block location cache refreshPolicy(); setBlockPlacementPolicy(namesystem, policy); // verify policy deletes the correct blocks. companion blocks should be // evenly distributed. fs.setReplication(sourcePath, (short) 1); DFSTestUtil.waitReplication(fs, sourcePath, (short) 1); Map counters = new HashMap(); refreshPolicy(); for (int i = 0; i < parityBlocks.size(); i++) { companionBlocks = getCompanionBlocks(namesystem, policy, parityBlocks.get(i).getBlock()); counters = BlockPlacementPolicyRaid.countCompanionBlocks(companionBlocks, false); Assert.assertTrue(counters.get(datanode1.getName()) >= 1 && counters.get(datanode1.getName()) <= 2); Assert.assertTrue(counters.get(datanode1.getName()) + counters.get(datanode2.getName()) == companionBlocks.size()); counters = BlockPlacementPolicyRaid.countCompanionBlocks(companionBlocks, true); Assert.assertTrue(counters.get(datanode1.getParent().getName()) >= 1 && counters.get(datanode1.getParent().getName()) <= 2); Assert.assertTrue(counters.get(datanode1.getParent().getName()) + counters.get(datanode2.getParent().getName()) == companionBlocks.size()); } } finally { if (cluster != null) { cluster.shutdown(); } }
5558 -1009652168apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Set the policy to default policy to place the block in the default way SATD_ADDED testGetCompanionBLocks() public void testGetCompanionBLocks() throws IOException setupCluster(); try { String file1 = "/dir/file1"; String file2 = "/raid/dir/file2"; String file3 = "/raidrs/dir/file3"; // Set the policy to default policy to place the block in the default way setBlockPlacementPolicy(namesystem, new BlockPlacementPolicyDefault(conf, namesystem, namesystem.clusterMap)); DFSTestUtil.createFile(fs, new Path(file1), 3, (short) 1, 0L); DFSTestUtil.createFile(fs, new Path(file2), 4, (short) 1, 0L); DFSTestUtil.createFile(fs, new Path(file3), 8, (short) 1, 0L); Collection companionBlocks; companionBlocks = getCompanionBlocks(namesystem, policy, getBlocks(namesystem, file1).get(0).getBlock()); Assert.assertTrue(companionBlocks == null || companionBlocks.size() == 0); companionBlocks = getCompanionBlocks(namesystem, policy, getBlocks(namesystem, file1).get(2).getBlock()); Assert.assertTrue(companionBlocks == null || companionBlocks.size() == 0); companionBlocks = getCompanionBlocks(namesystem, policy, getBlocks(namesystem, file2).get(0).getBlock()); Assert.assertEquals(1, companionBlocks.size()); companionBlocks = getCompanionBlocks(namesystem, policy, getBlocks(namesystem, file2).get(3).getBlock()); Assert.assertEquals(1, companionBlocks.size()); int rsParityLength = RaidNode.rsParityLength(conf); companionBlocks = getCompanionBlocks(namesystem, policy, getBlocks(namesystem, file3).get(0).getBlock()); Assert.assertEquals(rsParityLength, companionBlocks.size()); companionBlocks = getCompanionBlocks(namesystem, policy, getBlocks(namesystem, file3).get(4).getBlock()); Assert.assertEquals(rsParityLength, companionBlocks.size()); companionBlocks = getCompanionBlocks(namesystem, policy, getBlocks(namesystem, file3).get(6).getBlock()); Assert.assertEquals(2, companionBlocks.size()); } finally { if (cluster != null) { cluster.shutdown(); } }
5556 -1009652170apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None TODO: un-comment checkFullFile once the lease recovery is done SATD_ADDED checkFullFile(FileSystem, Path) private static void checkFullFile(FileSystem fs, Path p) throws IOException // TestFileCreation.checkFullFile(fs, p);
5555 -1009652171apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Print the Job Execution Statistics TODO: split to pring job, map/reduce task list and individual map/reduce task stats SATD_ADDED setJobConf(JobConf) void setJobConf(JobConf jobConf) this._jobConf = jobConf; // TODO: Add job conf to _job array
5553 -1009652173apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None better if we don't create the input format instance SATD_ADDED createJob(Configuration, Path[], Path, int, Shard[]) // set the starting generation for each shard // when a reduce task fails, a new reduce task // has to know where to re-start setShardGeneration(conf, shards); // iconf.set sets properties in conf IndexUpdateConfiguration iconf = new IndexUpdateConfiguration(conf); Shard.setIndexShards(iconf, shards); // MapTask.MapOutputBuffer uses JobContext.IO_SORT_MB to decide its max buffer size // (max buffer size = 1/2 * JobContext.IO_SORT_MB). // Here we half-en JobContext.IO_SORT_MB because we use the other half memory to // build an intermediate form/index in Combiner. iconf.setIOSortMB(iconf.getIOSortMB() / 2); // create the job configuration JobConf jobConf = new JobConf(conf, IndexUpdater.class); jobConf.setJobName(this.getClass().getName() + "_" + System.currentTimeMillis()); // provided by application FileInputFormat.setInputPaths(jobConf, inputPaths); FileOutputFormat.setOutputPath(jobConf, outputPath); jobConf.setNumMapTasks(numMapTasks); // already set shards jobConf.setNumReduceTasks(shards.length); jobConf.setInputFormat(iconf.getIndexInputFormatClass()); Path[] inputs = FileInputFormat.getInputPaths(jobConf); StringBuilder buffer = new StringBuilder(inputs[0].toString()); for (int i = 1; i < inputs.length; i++) { buffer.append(","); buffer.append(inputs[i].toString()); } LOG.info("mapred.input.dir = " + buffer.toString()); LOG.info("mapreduce.output.fileoutputformat.outputdir = " + FileOutputFormat.getOutputPath(jobConf).toString()); LOG.info("mapreduce.job.maps = " + jobConf.getNumMapTasks()); LOG.info("mapreduce.job.reduces = " + jobConf.getNumReduceTasks()); LOG.info(shards.length + " shards = " + iconf.getIndexShards()); // better if we don't create the input format instance LOG.info("mapred.input.format.class = " + jobConf.getInputFormat().getClass().getName()); // set by the system jobConf.setMapOutputKeyClass(IndexUpdateMapper.getMapOutputKeyClass()); jobConf.setMapOutputValueClass(IndexUpdateMapper.getMapOutputValueClass()); jobConf.setOutputKeyClass(IndexUpdateReducer.getOutputKeyClass()); jobConf.setOutputValueClass(IndexUpdateReducer.getOutputValueClass()); jobConf.setMapperClass(IndexUpdateMapper.class); jobConf.setPartitionerClass(IndexUpdatePartitioner.class); jobConf.setCombinerClass(IndexUpdateCombiner.class); jobConf.setReducerClass(IndexUpdateReducer.class); jobConf.setOutputFormat(IndexUpdateOutputFormat.class); return jobConf; JobConf createJob(Configuration conf, Path[] inputPaths, Path outputPath, int numMapTasks, Shard[] shards) throws IOException
5552 -1009652174apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None whether TT knows about this task sleep for a bit SATD_ADDED getFileSystemCounterNames(String) protected static String[] getFileSystemCounterNames(String uriScheme) String scheme = uriScheme.toUpperCase(); return new String[] { scheme + "_BYTES_READ", scheme + "_BYTES_WRITTEN" };
5551 -1009652175apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None report blocks once SATD_ADDED generateInputs(int[]) void generateInputs(int[] ignore) throws IOException final FSNamesystem namesystem = nameNode.getNamesystem(); // start data-nodes; create a bunch of files; generate block reports. blockReportObject.generateInputs(ignore); // stop replication monitor namesystem.replthread.interrupt(); try { namesystem.replthread.join(); } catch (InterruptedException ei) { return; } // report blocks once int nrDatanodes = blockReportObject.getNumDatanodes(); for (int idx = 0; idx < nrDatanodes; idx++) { blockReportObject.executeOp(idx, 0, null); } // decommission data-nodes decommissionNodes(); // set node replication limit namesystem.setNodeReplicationLimit(nodeReplicationLimit);
5550 -1009652176apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Currently, authorization is not done for jobtracker page. So allow everyone to view it. SATD_ADDED testWebUIAuthorization() public void testWebUIAuthorization() throws Exception setupGroupsProvider(); Properties props = new Properties(); props.setProperty("hadoop.http.filter.initializers", DummyFilterInitializer.class.getName()); props.setProperty(MRConfig.MR_ACLS_ENABLED, String.valueOf(true)); props.setProperty("dfs.permissions.enabled", "false"); props.setProperty(JTConfig.PRIVATE_ACTIONS_KEY, "true"); props.setProperty(MRJobConfig.SETUP_CLEANUP_NEEDED, "false"); props.setProperty(MRConfig.MR_SUPERGROUP, "superGroup"); props.setProperty(MRConfig.MR_ADMINS, mrAdminUser + " " + mrAdminGroup); String[] queueNames = { "default" }; String[] submitAclStrings = { jobSubmitter + "," + jobSubmitter1 + "," + jobSubmitter2 + "," + jobSubmitter3 }; String[] adminsAclStrings = new String[] { qAdmin }; startCluster(props, queueNames, submitAclStrings, adminsAclStrings); MiniMRCluster cluster = getMRCluster(); int infoPort = cluster.getJobTrackerRunner().getJobTrackerInfoPort(); JobConf clusterConf = cluster.createJobConf(); JobConf conf = new JobConf(clusterConf); conf.set(MRJobConfig.JOB_ACL_VIEW_JOB, viewColleague + " group3"); // Let us add group1 and group3 to modify-job-acl. So modifyColleague and // viewAndModifyColleague will be able to modify the job conf.set(MRJobConfig.JOB_ACL_MODIFY_JOB, " group1,group3"); Job job = startSleepJobAsUser(jobSubmitter, conf); org.apache.hadoop.mapreduce.JobID jobid = job.getJobID(); String jtURL = "http://localhost:" + infoPort; String jobTrackerJSP = jtURL + "/jobtracker.jsp?a=b"; try { // Currently, authorization is not done for jobtracker page. So allow // everyone to view it. validateJobTrackerJSPAccess(jtURL); validateJobDetailsJSPAccess(jobid, jtURL); validateTaskGraphServletAccess(jobid, jtURL); validateJobTasksJSPAccess(jobid, jtURL); validateJobConfJSPAccess(jobid, jtURL); validateJobFailuresJSPAccess(jobid, jtURL); valiateJobBlacklistedTrackerJSPAccess(jobid, jtURL); validateJobTrackerJSPSetPriorityAction(jobid, jtURL); // Wait for the tip to start so as to test task related JSP TaskID tipId = getTIPId(cluster, jobid); validateTaskStatsJSPAccess(jobid, jtURL, tipId); validateTaskDetailsJSPAccess(jobid, jtURL, tipId); validateJobTrackerJSPKillJobAction(jobid, jtURL); } finally { if (!job.isComplete()) { // kill the job(as jobSubmitter) if needed LOG.info("Killing job " + jobid + " from finally block"); assertEquals(HttpURLConnection.HTTP_OK, getHttpStatusCode(jobTrackerJSP + "&killJobs=true&jobCheckBox=" + jobid.toString(), jobSubmitter, "GET")); } } // validate killJob of jobdetails.jsp validateJobDetailsJSPKillJob(cluster, clusterConf, jtURL); // validate killJob of jobtracker.jsp as users viewAndModifyColleague, // jobSubmitter, mrOwner, mrAdmin and superGroupMember confirmJobTrackerJSPKillJobAsUser(cluster, conf, jtURL, viewAndModifyColleague); confirmJobTrackerJSPKillJobAsUser(cluster, conf, jtURL, jobSubmitter); confirmJobTrackerJSPKillJobAsUser(cluster, conf, jtURL, mrOwner); confirmJobTrackerJSPKillJobAsUser(cluster, conf, jtURL, superGroupMember); confirmJobTrackerJSPKillJobAsUser(cluster, conf, jtURL, mrAdminUser); confirmJobTrackerJSPKillJobAsUser(cluster, conf, jtURL, mrAdminGroupMember); confirmJobTrackerJSPKillJobAsUser(cluster, conf, jtURL, qAdmin); // validate killing of multiple jobs using jobtracker jsp and check // if all the jobs which can be killed by user are actually the ones that // got killed validateKillMultipleJobs(cluster, conf, jtURL);
5549 -1009652177apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Maybe the server is in unsecure mode (that's bad but okay) SATD_ADDED getDateFormat() public static final SimpleDateFormat getDateFormat() final SimpleDateFormat df = new SimpleDateFormat(HFTP_DATE_FORMAT); df.setTimeZone(TimeZone.getTimeZone(HFTP_TIMEZONE)); return df;
5548 -1009652178apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None TODO SATD_ADDED performFinish() public HadoopServer performFinish() try { if (this.original == null) { // New location Display.getDefault().syncExec(new Runnable() { public void run() { ServerRegistry.getInstance().addServer(HadoopLocationWizard.this.location); } }); return this.location; } else { // Update location final String originalName = this.original.getLocationName(); this.original.load(this.location); Display.getDefault().syncExec(new Runnable() { public void run() { ServerRegistry.getInstance().updateServer(originalName, HadoopLocationWizard.this.location); } }); return this.original; } } catch (Exception e) { e.printStackTrace(); setMessage("Invalid server location values", IMessageProvider.ERROR); return null; }
5547 -1009652179apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None If distributed cache is modified in dfs between two job runs, it can be present more than once in any of the task tracker, in which job ran. SATD_ADDED testDistributedCache() public void testDistributedCache() throws Exception Configuration conf = new Configuration(cluster.getConf()); JTProtocol wovenClient = cluster.getJTClient().getProxy(); // This counter will check for count of a loop, // which might become infinite. int count = 0; // This boolean will decide whether to run job again boolean continueLoop = true; // counter for job Loop int countLoop = 0; // This counter increases with all the tasktrackers in which tasks ran int taskTrackerCounter = 0; // This will store all the tasktrackers in which tasks ran ArrayList taskTrackerCollection = new ArrayList(); // This boolean tells if two tasks ran onteh same tasktracker or not boolean taskTrackerFound = false; do { SleepJob job = new SleepJob(); job.setConf(conf); Job slpJob = job.createJob(5, 1, 1000, 1000, 100, 100); // Before starting, Modify the file String input = "This will be the content of\n" + "distributed cache\n"; // Creating the path with the file DataOutputStream file = UtilsForTests.createTmpFileDFS(dfs, URIPATH, permission, input); DistributedCache.createSymlink(conf); URI uri = URI.create(uriPath); DistributedCache.addCacheFile(uri, conf); JobConf jconf = new JobConf(conf); // Controls the job till all verification is done FinishTaskControlAction.configureControlActionForJob(conf); slpJob.submit(); // Submitting the job RunningJob rJob = cluster.getJTClient().getClient().getJob(org.apache.hadoop.mapred.JobID.downgrade(slpJob.getJobID())); // counter for job Loop countLoop++; TTClient tClient = null; JobInfo jInfo = wovenClient.getJobInfo(rJob.getID()); LOG.info("jInfo is :" + jInfo); // Assert if jobInfo is null Assert.assertNotNull("jobInfo is null", jInfo); // Wait for the job to start running. count = 0; while (jInfo.getStatus().getRunState() != JobStatus.RUNNING) { UtilsForTests.waitFor(10000); count++; jInfo = wovenClient.getJobInfo(rJob.getID()); // If the count goes beyond a point, then break; This is to avoid // infinite loop under unforeseen circumstances. Testcase will anyway // fail later. if (count > 10) { Assert.fail("job has not reached running state for more than" + "100 seconds. Failing at this point"); } } LOG.info("job id is :" + rJob.getID().toString()); TaskInfo[] taskInfos = cluster.getJTClient().getProxy().getTaskInfo(rJob.getID()); boolean distCacheFileIsFound; for (TaskInfo taskInfo : taskInfos) { distCacheFileIsFound = false; String[] taskTrackers = taskInfo.getTaskTrackers(); for (String taskTracker : taskTrackers) { // Formatting tasktracker to get just its FQDN taskTracker = UtilsForTests.getFQDNofTT(taskTracker); LOG.info("taskTracker is :" + taskTracker); // The tasktrackerFound variable is initialized taskTrackerFound = false; // This will be entered from the second job onwards if (countLoop > 1) { if (taskTracker != null) { continueLoop = taskTrackerCollection.contains(taskTracker); } if (continueLoop) { taskTrackerFound = true; } } // Collecting the tasktrackers if (taskTracker != null) taskTrackerCollection.add(taskTracker); // we have loopped through two times to look for task // getting submitted on same tasktrackers.The same tasktracker // for subsequent jobs was not hit maybe because of many number // of tasktrackers. So, testcase has to stop here. if (countLoop > 1) { continueLoop = false; } tClient = cluster.getTTClient(taskTracker); // tClient maybe null because the task is already dead. Ex: setup if (tClient == null) { continue; } String[] localDirs = tClient.getMapredLocalDirs(); int distributedFileCount = 0; // Go to every single path for (String localDir : localDirs) { // Public Distributed cache will always be stored under // mapre.local.dir/tasktracker/archive localDir = localDir + Path.SEPARATOR + TaskTracker.getPublicDistributedCacheDir(); LOG.info("localDir is : " + localDir); // Get file status of all the directories // and files under that path. FileStatus[] fileStatuses = tClient.listStatus(localDir, true, true); for (FileStatus fileStatus : fileStatuses) { Path path = fileStatus.getPath(); LOG.info("path is :" + path.toString()); // Checking if the received path ends with // the distributed filename distCacheFileIsFound = (path.toString()).endsWith(distributedFileName); // If file is found, check for its permission. // Since the file is found break out of loop if (distCacheFileIsFound) { LOG.info("PATH found is :" + path.toString()); distributedFileCount++; String filename = path.getName(); FsPermission fsPerm = fileStatus.getPermission(); Assert.assertTrue("File Permission is not 777", fsPerm.equals(new FsPermission("777"))); } } } LOG.debug("The distributed FileCount is :" + distributedFileCount); LOG.debug("The taskTrackerFound is :" + taskTrackerFound); // If distributed cache is modified in dfs // between two job runs, it can be present more than once // in any of the task tracker, in which job ran. if (distributedFileCount != 2 && taskTrackerFound) { Assert.fail("The distributed cache file has to be two. " + "But found was " + distributedFileCount); } else if (distributedFileCount > 1 && !taskTrackerFound) { Assert.fail("The distributed cache file cannot more than one." + " But found was " + distributedFileCount); } else if (distributedFileCount < 1) Assert.fail("The distributed cache file is less than one. " + "But found was " + distributedFileCount); if (!distCacheFileIsFound) { Assert.assertEquals("The distributed cache file does not exist", distCacheFileIsFound, false); } } } // Allow the job to continue through MR control job. for (TaskInfo taskInfoRemaining : taskInfos) { FinishTaskControlAction action = new FinishTaskControlAction(TaskID.downgrade(taskInfoRemaining.getTaskID())); Collection tts = cluster.getTTClients(); for (TTClient cli : tts) { cli.getProxy().sendAction(action); } } // Killing the job because all the verification needed // for this testcase is completed. rJob.killJob(); // Waiting for 3 seconds for cleanup to start Thread.sleep(3000); // Getting the last cleanup task's tasktracker also, as // distributed cache gets uploaded even during cleanup. TaskInfo[] myTaskInfos = wovenClient.getTaskInfo(rJob.getID()); if (myTaskInfos != null) { for (TaskInfo info : myTaskInfos) { if (info.isSetupOrCleanup()) { String[] taskTrackers = info.getTaskTrackers(); for (String taskTracker : taskTrackers) { // Formatting tasktracker to get just its FQDN taskTracker = UtilsForTests.getFQDNofTT(taskTracker); LOG.info("taskTracker is :" + taskTracker); // Collecting the tasktrackers if (taskTracker != null) taskTrackerCollection.add(taskTracker); } } } } // Making sure that the job is complete. while (jInfo != null && !jInfo.getStatus().isJobComplete()) { Thread.sleep(10000); jInfo = wovenClient.getJobInfo(rJob.getID()); } } while (continueLoop);
5546 -1009652180apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None tClient maybe null because the task is already dead. Ex: setup SATD_ADDED testDistributedCache() public void testDistributedCache() throws Exception Configuration conf = new Configuration(cluster.getConf()); JTProtocol wovenClient = cluster.getJTClient().getProxy(); // This counter will check for count of a loop, // which might become infinite. int count = 0; // This boolean will decide whether to run job again boolean continueLoop = true; // counter for job Loop int countLoop = 0; // This counter increases with all the tasktrackers in which tasks ran int taskTrackerCounter = 0; // This will store all the tasktrackers in which tasks ran ArrayList taskTrackerCollection = new ArrayList(); // This boolean tells if two tasks ran onteh same tasktracker or not boolean taskTrackerFound = false; do { SleepJob job = new SleepJob(); job.setConf(conf); Job slpJob = job.createJob(5, 1, 1000, 1000, 100, 100); // Before starting, Modify the file String input = "This will be the content of\n" + "distributed cache\n"; // Creating the path with the file DataOutputStream file = UtilsForTests.createTmpFileDFS(dfs, URIPATH, permission, input); DistributedCache.createSymlink(conf); URI uri = URI.create(uriPath); DistributedCache.addCacheFile(uri, conf); JobConf jconf = new JobConf(conf); // Controls the job till all verification is done FinishTaskControlAction.configureControlActionForJob(conf); slpJob.submit(); // Submitting the job RunningJob rJob = cluster.getJTClient().getClient().getJob(org.apache.hadoop.mapred.JobID.downgrade(slpJob.getJobID())); // counter for job Loop countLoop++; TTClient tClient = null; JobInfo jInfo = wovenClient.getJobInfo(rJob.getID()); LOG.info("jInfo is :" + jInfo); // Assert if jobInfo is null Assert.assertNotNull("jobInfo is null", jInfo); // Wait for the job to start running. count = 0; while (jInfo.getStatus().getRunState() != JobStatus.RUNNING) { UtilsForTests.waitFor(10000); count++; jInfo = wovenClient.getJobInfo(rJob.getID()); // If the count goes beyond a point, then break; This is to avoid // infinite loop under unforeseen circumstances. Testcase will anyway // fail later. if (count > 10) { Assert.fail("job has not reached running state for more than" + "100 seconds. Failing at this point"); } } LOG.info("job id is :" + rJob.getID().toString()); TaskInfo[] taskInfos = cluster.getJTClient().getProxy().getTaskInfo(rJob.getID()); boolean distCacheFileIsFound; for (TaskInfo taskInfo : taskInfos) { distCacheFileIsFound = false; String[] taskTrackers = taskInfo.getTaskTrackers(); for (String taskTracker : taskTrackers) { // Formatting tasktracker to get just its FQDN taskTracker = UtilsForTests.getFQDNofTT(taskTracker); LOG.info("taskTracker is :" + taskTracker); // The tasktrackerFound variable is initialized taskTrackerFound = false; // This will be entered from the second job onwards if (countLoop > 1) { if (taskTracker != null) { continueLoop = taskTrackerCollection.contains(taskTracker); } if (continueLoop) { taskTrackerFound = true; } } // Collecting the tasktrackers if (taskTracker != null) taskTrackerCollection.add(taskTracker); // we have loopped through two times to look for task // getting submitted on same tasktrackers.The same tasktracker // for subsequent jobs was not hit maybe because of many number // of tasktrackers. So, testcase has to stop here. if (countLoop > 1) { continueLoop = false; } tClient = cluster.getTTClient(taskTracker); // tClient maybe null because the task is already dead. Ex: setup if (tClient == null) { continue; } String[] localDirs = tClient.getMapredLocalDirs(); int distributedFileCount = 0; // Go to every single path for (String localDir : localDirs) { // Public Distributed cache will always be stored under // mapre.local.dir/tasktracker/archive localDir = localDir + Path.SEPARATOR + TaskTracker.getPublicDistributedCacheDir(); LOG.info("localDir is : " + localDir); // Get file status of all the directories // and files under that path. FileStatus[] fileStatuses = tClient.listStatus(localDir, true, true); for (FileStatus fileStatus : fileStatuses) { Path path = fileStatus.getPath(); LOG.info("path is :" + path.toString()); // Checking if the received path ends with // the distributed filename distCacheFileIsFound = (path.toString()).endsWith(distributedFileName); // If file is found, check for its permission. // Since the file is found break out of loop if (distCacheFileIsFound) { LOG.info("PATH found is :" + path.toString()); distributedFileCount++; String filename = path.getName(); FsPermission fsPerm = fileStatus.getPermission(); Assert.assertTrue("File Permission is not 777", fsPerm.equals(new FsPermission("777"))); } } } LOG.debug("The distributed FileCount is :" + distributedFileCount); LOG.debug("The taskTrackerFound is :" + taskTrackerFound); // If distributed cache is modified in dfs // between two job runs, it can be present more than once // in any of the task tracker, in which job ran. if (distributedFileCount != 2 && taskTrackerFound) { Assert.fail("The distributed cache file has to be two. " + "But found was " + distributedFileCount); } else if (distributedFileCount > 1 && !taskTrackerFound) { Assert.fail("The distributed cache file cannot more than one." + " But found was " + distributedFileCount); } else if (distributedFileCount < 1) Assert.fail("The distributed cache file is less than one. " + "But found was " + distributedFileCount); if (!distCacheFileIsFound) { Assert.assertEquals("The distributed cache file does not exist", distCacheFileIsFound, false); } } } // Allow the job to continue through MR control job. for (TaskInfo taskInfoRemaining : taskInfos) { FinishTaskControlAction action = new FinishTaskControlAction(TaskID.downgrade(taskInfoRemaining.getTaskID())); Collection tts = cluster.getTTClients(); for (TTClient cli : tts) { cli.getProxy().sendAction(action); } } // Killing the job because all the verification needed // for this testcase is completed. rJob.killJob(); // Waiting for 3 seconds for cleanup to start Thread.sleep(3000); // Getting the last cleanup task's tasktracker also, as // distributed cache gets uploaded even during cleanup. TaskInfo[] myTaskInfos = wovenClient.getTaskInfo(rJob.getID()); if (myTaskInfos != null) { for (TaskInfo info : myTaskInfos) { if (info.isSetupOrCleanup()) { String[] taskTrackers = info.getTaskTrackers(); for (String taskTracker : taskTrackers) { // Formatting tasktracker to get just its FQDN taskTracker = UtilsForTests.getFQDNofTT(taskTracker); LOG.info("taskTracker is :" + taskTracker); // Collecting the tasktrackers if (taskTracker != null) taskTrackerCollection.add(taskTracker); } } } } // Making sure that the job is complete. while (jInfo != null && !jInfo.getStatus().isJobComplete()) { Thread.sleep(10000); jInfo = wovenClient.getJobInfo(rJob.getID()); } } while (continueLoop);
5545 -1009652181apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None When step == 0, this loop starts as many map tasks it can wrt maxTasksPerJob When step == 1, this loop starts as many reduce tasks it can wrt maxTasksPerJob When step == 2, this loop starts as many map tasks it can When step == 3, this loop starts as many reduce tasks it can It may seem that we would improve this loop by queuing jobs we cannot start in steps 0 and 1 because of maxTasksPerJob, and using that queue in step 2 and 3. A first thing to notice is that the time with the current algorithm is logarithmic, because it is the sum of (p^k) for k from 1 to N, were N is the number of jobs and p is the probability for a job to not exceed limits The probability for the cache to be useful would be similar to p^N, that is 1/(e^N), whereas its size and the time spent to manage it would be in ln(N). So it is not a good idea. SATD_ADDED assignTasks(TaskTracker) public synchronized List assignTasks(TaskTracker taskTracker) throws IOException TaskTrackerStatus taskTrackerStatus = taskTracker.getStatus(); final int numTaskTrackers = taskTrackerManager.getClusterStatus().getTaskTrackers(); Collection jobQueue = jobQueueJobInProgressListener.getJobQueue(); Task task; /* Stats about the current taskTracker */ final int mapTasksNumber = taskTrackerStatus.countMapTasks(); final int reduceTasksNumber = taskTrackerStatus.countReduceTasks(); final int maximumMapTasksNumber = taskTrackerStatus.getMaxMapSlots(); final int maximumReduceTasksNumber = taskTrackerStatus.getMaxReduceSlots(); /* * Statistics about the whole cluster. Most are approximate because of * concurrency */ final int[] maxMapAndReduceLoad = getMaxMapAndReduceLoad(maximumMapTasksNumber, maximumReduceTasksNumber); final int maximumMapLoad = maxMapAndReduceLoad[0]; final int maximumReduceLoad = maxMapAndReduceLoad[1]; final int beginAtStep; /* * When step == 0, this loop starts as many map tasks it can wrt * maxTasksPerJob * When step == 1, this loop starts as many reduce tasks it can wrt * maxTasksPerJob * When step == 2, this loop starts as many map tasks it can * When step == 3, this loop starts as many reduce tasks it can * * It may seem that we would improve this loop by queuing jobs we cannot * start in steps 0 and 1 because of maxTasksPerJob, and using that queue * in step 2 and 3. * A first thing to notice is that the time with the current algorithm is * logarithmic, because it is the sum of (p^k) for k from 1 to N, were * N is the number of jobs and p is the probability for a job to not exceed * limits The probability for the cache to be useful would be similar to * p^N, that is 1/(e^N), whereas its size and the time spent to manage it * would be in ln(N). * So it is not a good idea. */ if (maxTasksPerJob != Long.MAX_VALUE) { beginAtStep = 0; } else { beginAtStep = 2; } List assignedTasks = new ArrayList(); scheduleTasks: for (int step = beginAtStep; step <= 3; ++step) { /* If we reached the maximum load for this step, go to the next */ if ((step == 0 || step == 2) && mapTasksNumber >= maximumMapLoad || (step == 1 || step == 3) && reduceTasksNumber >= maximumReduceLoad) { continue; } /* For each job, start its tasks */ synchronized (jobQueue) { for (JobInProgress job : jobQueue) { /* Ignore non running jobs */ if (job.getStatus().getRunState() != JobStatus.RUNNING) { continue; } /* Check that we're not exceeding the global limits */ if ((step == 0 || step == 1) && (job.runningMaps() + job.runningReduces() >= maxTasksPerJob)) { continue; } if (step == 0 || step == 2) { task = job.obtainNewMapTask(taskTrackerStatus, numTaskTrackers, taskTrackerManager.getNumberOfUniqueHosts()); } else { task = job.obtainNewReduceTask(taskTrackerStatus, numTaskTrackers, taskTrackerManager.getNumberOfUniqueHosts()); } if (task != null) { assignedTasks.add(task); break scheduleTasks; } } } } return assignedTasks;
5544 -1009652182apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None TODO fix this: make it more sophisticated!!! SATD_ADDED injectCriteria(String) public static boolean injectCriteria(String klassName) boolean trigger = false; // TODO fix this: make it more sophisticated!!! if (generator.nextFloat() < getProbability(klassName)) { trigger = true; } return trigger;
5543 -1009652183apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Set new default probability if specified through a system.property If neither is specified set default probability to DEFAULT_PROB SATD_ADDED injectCriteria(String) public static boolean injectCriteria(String klassName) boolean trigger = false; // TODO fix this: make it more sophisticated!!! if (generator.nextFloat() < getProbability(klassName)) { trigger = true; } return trigger;
5541 -1009652185apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None TODO: Support BINARY, VARBINARY, LONGVARBINARY, DISTINCT, CLOB, BLOB, ARRAY STRUCT, REF, DATALINK, and JAVA_OBJECT. SATD_ADDED getSplitter(int) protected DBSplitter getSplitter(int sqlDataType) switch(sqlDataType) { case Types.NUMERIC: case Types.DECIMAL: return new BigDecimalSplitter(); case Types.BIT: case Types.BOOLEAN: return new BooleanSplitter(); case Types.INTEGER: case Types.TINYINT: case Types.SMALLINT: case Types.BIGINT: return new IntegerSplitter(); case Types.REAL: case Types.FLOAT: case Types.DOUBLE: return new FloatSplitter(); case Types.CHAR: case Types.VARCHAR: case Types.LONGVARCHAR: return new TextSplitter(); case Types.DATE: case Types.TIME: case Types.TIMESTAMP: return new DateSplitter(); default: // TODO: Support BINARY, VARBINARY, LONGVARBINARY, DISTINCT, CLOB, BLOB, ARRAY // STRUCT, REF, DATALINK, and JAVA_OBJECT. return null; }
5540 -1009652186apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None scan all policies once every 5 second SATD_ADDED setUp(boolean) private void setUp(boolean doHar) throws IOException, ClassNotFoundException final int timeBeforeHar; if (doHar) { timeBeforeHar = 0; } else { timeBeforeHar = -1; } // Make sure data directory exists new File(TEST_DIR).mkdirs(); conf = new Configuration(); conf.set("raid.config.file", CONFIG_FILE); conf.setBoolean("raid.config.reload", true); conf.setLong("raid.config.reload.interval", RELOAD_INTERVAL); // scan all policies once every 5 second conf.setLong("raid.policy.rescan.interval", 5000); // make all deletions not go through Trash conf.set("fs.shell.delete.classname", "org.apache.hadoop.hdfs.DFSClient"); // do not use map-reduce cluster for Raiding conf.set("raid.classname", "org.apache.hadoop.raid.LocalRaidNode"); // use local block fixer conf.set("raid.blockfix.classname", "org.apache.hadoop.raid.LocalBlockFixer"); conf.set("raid.server.address", "localhost:0"); conf.setInt("hdfs.raid.stripeLength", STRIPE_BLOCKS); conf.set("hdfs.raid.locations", RAID_DIR); conf.setInt("dfs.corruptfilesreturned.max", 500); conf.setBoolean("dfs.permissions", false); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DATANODES).build(); cluster.waitActive(); dfs = (DistributedFileSystem) cluster.getFileSystem(); String namenode = dfs.getUri().toString(); FileSystem.setDefaultUri(conf, namenode); FileWriter fileWriter = new FileWriter(CONFIG_FILE); fileWriter.write("\n"); String str = " " + " " + " " + " xor " + " " + RAID_DIR + " " + " " + " targetReplication " + " 1 " + " after RAIDing, decrease the replication " + "factor of a file to this value. " + " " + " " + " metaReplication " + " 1 " + " replication factor of parity file " + " " + " " + " modTimePeriod " + " 2000 " + " time (milliseconds) after a file is modified " + "to make it a candidate for RAIDing " + " "; if (timeBeforeHar >= 0) { str += " " + " time_before_har " + " " + timeBeforeHar + " " + " amount of time waited before har'ing parity " + "files " + " "; } str += " " + " " + ""; fileWriter.write(str); fileWriter.close(); createTestFile(FILE_PATH0); createTestFile(FILE_PATH1); Path[] filePaths = { FILE_PATH0, FILE_PATH1 }; raidTestFiles(RAID_PATH, filePaths, doHar); clientConf = new Configuration(raidConf); clientConf.set("fs.hdfs.impl", "org.apache.hadoop.hdfs.DistributedRaidFileSystem"); clientConf.set("fs.raid.underlyingfs.impl", "org.apache.hadoop.hdfs.DistributedFileSystem"); // prepare shell and arguments shell = new RaidShell(clientConf); args = new String[2]; args[0] = "-fsck"; args[1] = DIR_PATH;
5539 -1009652187apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None The DN should consider itself dead SATD_ADDED testSuccessiveVolumeFailures() public void testSuccessiveVolumeFailures() throws Exception assumeTrue(!System.getProperty("os.name").startsWith("Windows")); // Bring up two more datanodes cluster.startDataNodes(conf, 2, true, null, null); cluster.waitActive(); /* * Calculate the total capacity of all the datanodes. Sleep for * three seconds to be sure the datanodes have had a chance to * heartbeat their capacities. */ Thread.sleep(WAIT_FOR_HEARTBEATS); FSNamesystem ns = cluster.getNamesystem(); long origCapacity = DFSTestUtil.getLiveDatanodeCapacity(ns); long dnCapacity = DFSTestUtil.getDatanodeCapacity(ns, 0); File dn1Vol1 = new File(dataDir, "data" + (2 * 0 + 1)); File dn2Vol1 = new File(dataDir, "data" + (2 * 1 + 1)); File dn3Vol1 = new File(dataDir, "data" + (2 * 2 + 1)); File dn3Vol2 = new File(dataDir, "data" + (2 * 2 + 2)); /* * Make the 1st volume directories on the first two datanodes * non-accessible. We don't make all three 1st volume directories * readonly since that would cause the entire pipeline to * fail. The client does not retry failed nodes even though * perhaps they could succeed because just a single volume failed. */ assertTrue("Couldn't chmod local vol", dn1Vol1.setExecutable(false)); assertTrue("Couldn't chmod local vol", dn2Vol1.setExecutable(false)); /* * Create file1 and wait for 3 replicas (ie all DNs can still * store a block). Then assert that all DNs are up, despite the * volume failures. */ Path file1 = new Path("/test1"); DFSTestUtil.createFile(fs, file1, 1024, (short) 3, 1L); DFSTestUtil.waitReplication(fs, file1, (short) 3); ArrayList dns = cluster.getDataNodes(); assertTrue("DN1 should be up", dns.get(0).isDatanodeUp()); assertTrue("DN2 should be up", dns.get(1).isDatanodeUp()); assertTrue("DN3 should be up", dns.get(2).isDatanodeUp()); /* * The metrics should confirm the volume failures. */ assertCounter("VolumeFailures", 1L, getMetrics(dns.get(0).getMetrics().name())); assertCounter("VolumeFailures", 1L, getMetrics(dns.get(1).getMetrics().name())); assertCounter("VolumeFailures", 0L, getMetrics(dns.get(2).getMetrics().name())); // Ensure we wait a sufficient amount of time assert (WAIT_FOR_HEARTBEATS * 10) > WAIT_FOR_DEATH; // Eventually the NN should report two volume failures DFSTestUtil.waitForDatanodeStatus(ns, 3, 0, 2, origCapacity - (1 * dnCapacity), WAIT_FOR_HEARTBEATS); /* * Now fail a volume on the third datanode. We should be able to get * three replicas since we've already identified the other failures. */ assertTrue("Couldn't chmod local vol", dn3Vol1.setExecutable(false)); Path file2 = new Path("/test2"); DFSTestUtil.createFile(fs, file2, 1024, (short) 3, 1L); DFSTestUtil.waitReplication(fs, file2, (short) 3); assertTrue("DN3 should still be up", dns.get(2).isDatanodeUp()); assertCounter("VolumeFailures", 1L, getMetrics(dns.get(2).getMetrics().name())); ArrayList live = new ArrayList(); ArrayList dead = new ArrayList(); ns.DFSNodesStatus(live, dead); live.clear(); dead.clear(); ns.DFSNodesStatus(live, dead); assertEquals("DN3 should have 1 failed volume", 1, live.get(2).getVolumeFailures()); /* * Once the datanodes have a chance to heartbeat their new capacity the * total capacity should be down by three volumes (assuming the host * did not grow or shrink the data volume while the test was running). */ dnCapacity = DFSTestUtil.getDatanodeCapacity(ns, 0); DFSTestUtil.waitForDatanodeStatus(ns, 3, 0, 3, origCapacity - (3 * dnCapacity), WAIT_FOR_HEARTBEATS); /* * Now fail the 2nd volume on the 3rd datanode. All its volumes * are now failed and so it should report two volume failures * and that it's no longer up. Only wait for two replicas since * we'll never get a third. */ assertTrue("Couldn't chmod local vol", dn3Vol2.setExecutable(false)); Path file3 = new Path("/test3"); DFSTestUtil.createFile(fs, file3, 1024, (short) 3, 1L); DFSTestUtil.waitReplication(fs, file3, (short) 2); // The DN should consider itself dead DFSTestUtil.waitForDatanodeDeath(dns.get(2)); // And report two failed volumes assertCounter("VolumeFailures", 2L, getMetrics(dns.get(2).getMetrics().name())); // The NN considers the DN dead DFSTestUtil.waitForDatanodeStatus(ns, 2, 1, 2, origCapacity - (4 * dnCapacity), WAIT_FOR_HEARTBEATS); /* * The datanode never tries to restore the failed volume, even if * it's subsequently repaired, but it should see this volume on * restart, so file creation should be able to succeed after * restoring the data directories and restarting the datanodes. */ assertTrue("Couldn't chmod local vol", dn1Vol1.setExecutable(true)); assertTrue("Couldn't chmod local vol", dn2Vol1.setExecutable(true)); assertTrue("Couldn't chmod local vol", dn3Vol1.setExecutable(true)); assertTrue("Couldn't chmod local vol", dn3Vol2.setExecutable(true)); cluster.restartDataNodes(); cluster.waitActive(); Path file4 = new Path("/test4"); DFSTestUtil.createFile(fs, file4, 1024, (short) 3, 1L); DFSTestUtil.waitReplication(fs, file4, (short) 3); /* * Eventually the capacity should be restored to its original value, * and that the volume failure count should be reported as zero by * both the metrics and the NN. */ DFSTestUtil.waitForDatanodeStatus(ns, 3, 0, 0, origCapacity, WAIT_FOR_HEARTBEATS);
5538 -1009652188apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Eventually the NN should report two volume failures SATD_ADDED testSuccessiveVolumeFailures() public void testSuccessiveVolumeFailures() throws Exception assumeTrue(!System.getProperty("os.name").startsWith("Windows")); // Bring up two more datanodes cluster.startDataNodes(conf, 2, true, null, null); cluster.waitActive(); /* * Calculate the total capacity of all the datanodes. Sleep for * three seconds to be sure the datanodes have had a chance to * heartbeat their capacities. */ Thread.sleep(WAIT_FOR_HEARTBEATS); FSNamesystem ns = cluster.getNamesystem(); long origCapacity = DFSTestUtil.getLiveDatanodeCapacity(ns); long dnCapacity = DFSTestUtil.getDatanodeCapacity(ns, 0); File dn1Vol1 = new File(dataDir, "data" + (2 * 0 + 1)); File dn2Vol1 = new File(dataDir, "data" + (2 * 1 + 1)); File dn3Vol1 = new File(dataDir, "data" + (2 * 2 + 1)); File dn3Vol2 = new File(dataDir, "data" + (2 * 2 + 2)); /* * Make the 1st volume directories on the first two datanodes * non-accessible. We don't make all three 1st volume directories * readonly since that would cause the entire pipeline to * fail. The client does not retry failed nodes even though * perhaps they could succeed because just a single volume failed. */ assertTrue("Couldn't chmod local vol", dn1Vol1.setExecutable(false)); assertTrue("Couldn't chmod local vol", dn2Vol1.setExecutable(false)); /* * Create file1 and wait for 3 replicas (ie all DNs can still * store a block). Then assert that all DNs are up, despite the * volume failures. */ Path file1 = new Path("/test1"); DFSTestUtil.createFile(fs, file1, 1024, (short) 3, 1L); DFSTestUtil.waitReplication(fs, file1, (short) 3); ArrayList dns = cluster.getDataNodes(); assertTrue("DN1 should be up", dns.get(0).isDatanodeUp()); assertTrue("DN2 should be up", dns.get(1).isDatanodeUp()); assertTrue("DN3 should be up", dns.get(2).isDatanodeUp()); /* * The metrics should confirm the volume failures. */ assertCounter("VolumeFailures", 1L, getMetrics(dns.get(0).getMetrics().name())); assertCounter("VolumeFailures", 1L, getMetrics(dns.get(1).getMetrics().name())); assertCounter("VolumeFailures", 0L, getMetrics(dns.get(2).getMetrics().name())); // Ensure we wait a sufficient amount of time assert (WAIT_FOR_HEARTBEATS * 10) > WAIT_FOR_DEATH; // Eventually the NN should report two volume failures DFSTestUtil.waitForDatanodeStatus(ns, 3, 0, 2, origCapacity - (1 * dnCapacity), WAIT_FOR_HEARTBEATS); /* * Now fail a volume on the third datanode. We should be able to get * three replicas since we've already identified the other failures. */ assertTrue("Couldn't chmod local vol", dn3Vol1.setExecutable(false)); Path file2 = new Path("/test2"); DFSTestUtil.createFile(fs, file2, 1024, (short) 3, 1L); DFSTestUtil.waitReplication(fs, file2, (short) 3); assertTrue("DN3 should still be up", dns.get(2).isDatanodeUp()); assertCounter("VolumeFailures", 1L, getMetrics(dns.get(2).getMetrics().name())); ArrayList live = new ArrayList(); ArrayList dead = new ArrayList(); ns.DFSNodesStatus(live, dead); live.clear(); dead.clear(); ns.DFSNodesStatus(live, dead); assertEquals("DN3 should have 1 failed volume", 1, live.get(2).getVolumeFailures()); /* * Once the datanodes have a chance to heartbeat their new capacity the * total capacity should be down by three volumes (assuming the host * did not grow or shrink the data volume while the test was running). */ dnCapacity = DFSTestUtil.getDatanodeCapacity(ns, 0); DFSTestUtil.waitForDatanodeStatus(ns, 3, 0, 3, origCapacity - (3 * dnCapacity), WAIT_FOR_HEARTBEATS); /* * Now fail the 2nd volume on the 3rd datanode. All its volumes * are now failed and so it should report two volume failures * and that it's no longer up. Only wait for two replicas since * we'll never get a third. */ assertTrue("Couldn't chmod local vol", dn3Vol2.setExecutable(false)); Path file3 = new Path("/test3"); DFSTestUtil.createFile(fs, file3, 1024, (short) 3, 1L); DFSTestUtil.waitReplication(fs, file3, (short) 2); // The DN should consider itself dead DFSTestUtil.waitForDatanodeDeath(dns.get(2)); // And report two failed volumes assertCounter("VolumeFailures", 2L, getMetrics(dns.get(2).getMetrics().name())); // The NN considers the DN dead DFSTestUtil.waitForDatanodeStatus(ns, 2, 1, 2, origCapacity - (4 * dnCapacity), WAIT_FOR_HEARTBEATS); /* * The datanode never tries to restore the failed volume, even if * it's subsequently repaired, but it should see this volume on * restart, so file creation should be able to succeed after * restoring the data directories and restarting the datanodes. */ assertTrue("Couldn't chmod local vol", dn1Vol1.setExecutable(true)); assertTrue("Couldn't chmod local vol", dn2Vol1.setExecutable(true)); assertTrue("Couldn't chmod local vol", dn3Vol1.setExecutable(true)); assertTrue("Couldn't chmod local vol", dn3Vol2.setExecutable(true)); cluster.restartDataNodes(); cluster.waitActive(); Path file4 = new Path("/test4"); DFSTestUtil.createFile(fs, file4, 1024, (short) 3, 1L); DFSTestUtil.waitReplication(fs, file4, (short) 3); /* * Eventually the capacity should be restored to its original value, * and that the volume failure count should be reported as zero by * both the metrics and the NN. */ DFSTestUtil.waitForDatanodeStatus(ns, 3, 0, 0, origCapacity, WAIT_FOR_HEARTBEATS);
5537 -1009652189apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Make the 1st volume directories on the first two datanodes non-accessible. We don't make all three 1st volume directories readonly since that would cause the entire pipeline to fail. The client does not retry failed nodes even though perhaps they could succeed because just a single volume failed. SATD_ADDED testSuccessiveVolumeFailures() public void testSuccessiveVolumeFailures() throws Exception assumeTrue(!System.getProperty("os.name").startsWith("Windows")); // Bring up two more datanodes cluster.startDataNodes(conf, 2, true, null, null); cluster.waitActive(); /* * Calculate the total capacity of all the datanodes. Sleep for * three seconds to be sure the datanodes have had a chance to * heartbeat their capacities. */ Thread.sleep(WAIT_FOR_HEARTBEATS); FSNamesystem ns = cluster.getNamesystem(); long origCapacity = DFSTestUtil.getLiveDatanodeCapacity(ns); long dnCapacity = DFSTestUtil.getDatanodeCapacity(ns, 0); File dn1Vol1 = new File(dataDir, "data" + (2 * 0 + 1)); File dn2Vol1 = new File(dataDir, "data" + (2 * 1 + 1)); File dn3Vol1 = new File(dataDir, "data" + (2 * 2 + 1)); File dn3Vol2 = new File(dataDir, "data" + (2 * 2 + 2)); /* * Make the 1st volume directories on the first two datanodes * non-accessible. We don't make all three 1st volume directories * readonly since that would cause the entire pipeline to * fail. The client does not retry failed nodes even though * perhaps they could succeed because just a single volume failed. */ assertTrue("Couldn't chmod local vol", dn1Vol1.setExecutable(false)); assertTrue("Couldn't chmod local vol", dn2Vol1.setExecutable(false)); /* * Create file1 and wait for 3 replicas (ie all DNs can still * store a block). Then assert that all DNs are up, despite the * volume failures. */ Path file1 = new Path("/test1"); DFSTestUtil.createFile(fs, file1, 1024, (short) 3, 1L); DFSTestUtil.waitReplication(fs, file1, (short) 3); ArrayList dns = cluster.getDataNodes(); assertTrue("DN1 should be up", dns.get(0).isDatanodeUp()); assertTrue("DN2 should be up", dns.get(1).isDatanodeUp()); assertTrue("DN3 should be up", dns.get(2).isDatanodeUp()); /* * The metrics should confirm the volume failures. */ assertCounter("VolumeFailures", 1L, getMetrics(dns.get(0).getMetrics().name())); assertCounter("VolumeFailures", 1L, getMetrics(dns.get(1).getMetrics().name())); assertCounter("VolumeFailures", 0L, getMetrics(dns.get(2).getMetrics().name())); // Ensure we wait a sufficient amount of time assert (WAIT_FOR_HEARTBEATS * 10) > WAIT_FOR_DEATH; // Eventually the NN should report two volume failures DFSTestUtil.waitForDatanodeStatus(ns, 3, 0, 2, origCapacity - (1 * dnCapacity), WAIT_FOR_HEARTBEATS); /* * Now fail a volume on the third datanode. We should be able to get * three replicas since we've already identified the other failures. */ assertTrue("Couldn't chmod local vol", dn3Vol1.setExecutable(false)); Path file2 = new Path("/test2"); DFSTestUtil.createFile(fs, file2, 1024, (short) 3, 1L); DFSTestUtil.waitReplication(fs, file2, (short) 3); assertTrue("DN3 should still be up", dns.get(2).isDatanodeUp()); assertCounter("VolumeFailures", 1L, getMetrics(dns.get(2).getMetrics().name())); ArrayList live = new ArrayList(); ArrayList dead = new ArrayList(); ns.DFSNodesStatus(live, dead); live.clear(); dead.clear(); ns.DFSNodesStatus(live, dead); assertEquals("DN3 should have 1 failed volume", 1, live.get(2).getVolumeFailures()); /* * Once the datanodes have a chance to heartbeat their new capacity the * total capacity should be down by three volumes (assuming the host * did not grow or shrink the data volume while the test was running). */ dnCapacity = DFSTestUtil.getDatanodeCapacity(ns, 0); DFSTestUtil.waitForDatanodeStatus(ns, 3, 0, 3, origCapacity - (3 * dnCapacity), WAIT_FOR_HEARTBEATS); /* * Now fail the 2nd volume on the 3rd datanode. All its volumes * are now failed and so it should report two volume failures * and that it's no longer up. Only wait for two replicas since * we'll never get a third. */ assertTrue("Couldn't chmod local vol", dn3Vol2.setExecutable(false)); Path file3 = new Path("/test3"); DFSTestUtil.createFile(fs, file3, 1024, (short) 3, 1L); DFSTestUtil.waitReplication(fs, file3, (short) 2); // The DN should consider itself dead DFSTestUtil.waitForDatanodeDeath(dns.get(2)); // And report two failed volumes assertCounter("VolumeFailures", 2L, getMetrics(dns.get(2).getMetrics().name())); // The NN considers the DN dead DFSTestUtil.waitForDatanodeStatus(ns, 2, 1, 2, origCapacity - (4 * dnCapacity), WAIT_FOR_HEARTBEATS); /* * The datanode never tries to restore the failed volume, even if * it's subsequently repaired, but it should see this volume on * restart, so file creation should be able to succeed after * restoring the data directories and restarting the datanodes. */ assertTrue("Couldn't chmod local vol", dn1Vol1.setExecutable(true)); assertTrue("Couldn't chmod local vol", dn2Vol1.setExecutable(true)); assertTrue("Couldn't chmod local vol", dn3Vol1.setExecutable(true)); assertTrue("Couldn't chmod local vol", dn3Vol2.setExecutable(true)); cluster.restartDataNodes(); cluster.waitActive(); Path file4 = new Path("/test4"); DFSTestUtil.createFile(fs, file4, 1024, (short) 3, 1L); DFSTestUtil.waitReplication(fs, file4, (short) 3); /* * Eventually the capacity should be restored to its original value, * and that the volume failure count should be reported as zero by * both the metrics and the NN. */ DFSTestUtil.waitForDatanodeStatus(ns, 3, 0, 0, origCapacity, WAIT_FOR_HEARTBEATS);
5536 -1009652190apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None XXX is this necessary? SATD_ADDED close() public void close() throws IOException mp
5535 -1009652191apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None If TT has Map and Reduce slot free, we assign 1 map and 1 reduce We base decision on how much is needed versus how much is used SATD_ADDED assignTasks(TaskTracker) public synchronized List assignTasks(TaskTracker taskTracker) throws IOException TaskLookupResult tlr; TaskTrackerStatus taskTrackerStatus = taskTracker.getStatus(); List result = new ArrayList(); /* * If TT has Map and Reduce slot free, we assign 1 map and 1 reduce * We base decision on how much is needed * versus how much is used */ ClusterStatus c = taskTrackerManager.getClusterStatus(); int mapClusterCapacity = c.getMaxMapTasks(); int reduceClusterCapacity = c.getMaxReduceTasks(); int maxMapSlots = taskTrackerStatus.getMaxMapSlots(); int currentMapSlots = taskTrackerStatus.countOccupiedMapSlots(); int maxReduceSlots = taskTrackerStatus.getMaxReduceSlots(); int currentReduceSlots = taskTrackerStatus.countOccupiedReduceSlots(); if (LOG.isDebugEnabled()) { LOG.debug("TT asking for task, max maps=" + taskTrackerStatus.getMaxMapSlots() + ", run maps=" + taskTrackerStatus.countMapTasks() + ", max reds=" + taskTrackerStatus.getMaxReduceSlots() + ", run reds=" + taskTrackerStatus.countReduceTasks() + ", map cap=" + mapClusterCapacity + ", red cap = " + reduceClusterCapacity); } /* * update all our QSC objects. * This involves updating each qsC structure. This operation depends * on the number of running jobs in a queue, and some waiting jobs. If it * becomes expensive, do it once every few heartbeats only. */ updateContextObjects(mapClusterCapacity, reduceClusterCapacity); // make sure we get our map or reduce scheduling object to update its // collection of QSC objects too. if (maxReduceSlots > currentReduceSlots) { // reduce slot available , try to get a // reduce task tlr = reduceScheduler.assignTasks(taskTracker); if (TaskLookupResult.LookUpStatus.TASK_FOUND == tlr.getLookUpStatus()) { result.add(tlr.getTask()); } } if (maxMapSlots > currentMapSlots) { // map slot available , try to get a map task tlr = mapScheduler.assignTasks(taskTracker); if (TaskLookupResult.LookUpStatus.TASK_FOUND == tlr.getLookUpStatus()) { result.add(tlr.getTask()); } } return (result.isEmpty()) ? null : result;
5534 -1009652192apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None we do not 'reserve' slots on tasktrackers anymore since the user is already over the limit Note: some of the code from above is repeated here. This is on purpose as it improves overall readability. Note: we walk through jobs again. Some of these jobs, which weren't considered in the first pass, shouldn't be considered here again, but we still check for their viability to keep the code simple. In some cases, for high mem jobs that have nothing to run, we call obtainNewTask() unnecessarily. Should this be a problem, we can create a list of jobs to look at (those whose users were over limit) in the first pass and walk through that list only. SATD_ADDED toString() public String toString() // note that we do not call updateContextObjects() here for performance // reasons. This means that the data we print out may be slightly // stale. This data is updated whenever assignTasks() is called // If this doesn't happen, the data gets stale. If we see // this often, we may need to detect this situation and call // updateContextObjects(), or just call it each time. return scheduler.getDisplayInfo(queueName);
5533 -1009652193apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None look at how much capacity they've filled. Treat a queue with capacity=0 equivalent to a queue running at capacity SATD_ADDED toString() public String toString() // note that we do not call updateContextObjects() here for performance // reasons. This means that the data we print out may be slightly // stale. This data is updated whenever assignTasks() is called // If this doesn't happen, the data gets stale. If we see // this often, we may need to detect this situation and call // updateContextObjects(), or just call it each time. return scheduler.getDisplayInfo(queueName);
5532 -1009652194apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None FIXME Error dialog SATD_ADDED findLaunchConfiguration(IType, ILaunchConfigurationType) protected ILaunchConfiguration findLaunchConfiguration(IType type, ILaunchConfigurationType configType) // Find an existing or create a launch configuration (Standard way) ILaunchConfiguration iConf = super.findLaunchConfiguration(type, configType); if (iConf == null) iConf = super.createConfiguration(type); ILaunchConfigurationWorkingCopy iConfWC; try { /* * Tune the default launch configuration: setup run-time classpath * manually */ iConfWC = iConf.getWorkingCopy(); iConfWC.setAttribute(IJavaLaunchConfigurationConstants.ATTR_DEFAULT_CLASSPATH, false); List classPath = new ArrayList(); IResource resource = type.getResource(); IJavaProject project = (IJavaProject) resource.getProject().getNature(JavaCore.NATURE_ID); IRuntimeClasspathEntry cpEntry = JavaRuntime.newDefaultProjectClasspathEntry(project); classPath.add(0, cpEntry.getMemento()); iConfWC.setAttribute(IJavaLaunchConfigurationConstants.ATTR_CLASSPATH, classPath); } catch (CoreException e) { e.printStackTrace(); // FIXME Error dialog return null; } /* * Update the selected configuration with a specific Hadoop location * target */ IResource resource = type.getResource(); if (!(resource instanceof IFile)) return null; RunOnHadoopWizard wizard = new RunOnHadoopWizard((IFile) resource, iConfWC); WizardDialog dialog = new WizardDialog(Display.getDefault().getActiveShell(), wizard); dialog.create(); dialog.setBlockOnOpen(true); if (dialog.open() != WizardDialog.OK) return null; try { // Only save if some configuration is different. if (!iConfWC.contentsEqual(iConf)) iConfWC.doSave(); } catch (CoreException e) { e.printStackTrace(); // FIXME Error dialog return null; } return iConfWC;
5531 -1009652195apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None TODO: move it to DatanodeID once HADOOP-2797 has been committed SATD_ADDED readFields(DataInput) public void readFields(DataInput in) throws IOException super.readFields(in); // TODO: move it to DatanodeID once HADOOP-2797 has been committed this.ipcPort = in.readShort() & 0x0000ffff; storageInfo.readFields(in); exportedKeys.readFields(in);
5530 -1009652196apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None TODO: move it to DatanodeID once HADOOP-2797 has been committed SATD_ADDED write(DataOutput) public void write(DataOutput out) throws IOException super.write(out); // TODO: move it to DatanodeID once HADOOP-2797 has been committed out.writeShort(ipcPort); storageInfo.write(out); exportedKeys.write(out);
5528 -1009652198apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None test if jobs with no tasks (0 maps, 0 red) update the listener properly SATD_ADDED testSuccessfulJob(JobConf, MyListener) private void testSuccessfulJob(JobConf job, MyListener myListener) throws Exception LOG.info("Testing job-success"); Path inDir = new Path(TEST_ROOT_DIR + "/jiplistenerjob/input"); Path outDir = new Path(TEST_ROOT_DIR + "/jiplistenerjob/output"); job.setNumMapTasks(1); job.setNumReduceTasks(0); // submit the job RunningJob rJob = UtilsForTests.runJobSucceed(job, inDir, outDir); // wait for the job to be successful rJob.waitForCompletion(); // check if the job success was notified assertFalse("Missing event notification for a successful job", myListener.contains(rJob.getID())); // check if successful assertEquals("Job failed!", JobStatus.SUCCEEDED, rJob.getJobState()); // test if 0-task jobs with setup-cleanup works fine LOG.info("Testing job with no task job with setup and cleanup"); job.setNumMapTasks(0); job.setNumReduceTasks(0); outDir = new Path(TEST_ROOT_DIR + "/jiplistenerjob/output-no-tasks"); // submit the job rJob = UtilsForTests.runJobSucceed(job, inDir, outDir); // wait for the job to be successful rJob.waitForCompletion(); // check if the job success was notified assertFalse("Missing event notification for a successful job with no tasks", myListener.contains(rJob.getID(), true)); // check if successful assertEquals("Job failed!", JobStatus.SUCCEEDED, rJob.getJobState()); // test if jobs with no tasks (0 maps, 0 red) update the listener properly LOG.info("Testing job with no-set-cleanup no task"); outDir = new Path(TEST_ROOT_DIR + "/jiplistenerjob/output-no-tasks-no-set"); Job j = MapReduceTestUtil.createJob(mr.createJobConf(), inDir, outDir, 0, 0); j.setJobSetupCleanupNeeded(false); j.setOutputFormatClass(TestNoJobSetupCleanup.MyOutputFormat.class); j.submit(); j.waitForCompletion(true); JobID id = JobID.downgrade(j.getJobID()); // check if the job is in the waiting queue assertFalse("Missing event notification on no-set-cleanup no task job", myListener.contains(id, true)); // check if the job is successful assertEquals("Job status doesnt reflect success", JobStatus.SUCCEEDED, rJob.getJobState());
5527 -1009652199apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None add the correction factor of 234 as the input split is also streamed SATD_ADDED runWordCount(MiniMRCluster, JobConf) public static void runWordCount(MiniMRCluster mr, JobConf jobConf) throws IOException LOG.info("runWordCount"); // Run a word count example // Keeping tasks that match this pattern String pattern = TaskAttemptID.getTaskAttemptIDsPattern(null, null, TaskType.MAP, 1, null); jobConf.setKeepTaskFilesPattern(pattern); TestResult result; final Path inDir = new Path("./wc/input"); final Path outDir = new Path("./wc/output"); String input = "The quick brown fox\nhas many silly\nred fox sox\n"; result = launchWordCount(jobConf, inDir, outDir, input, 3, 1); assertEquals("The\t1\nbrown\t1\nfox\t2\nhas\t1\nmany\t1\n" + "quick\t1\nred\t1\nsilly\t1\nsox\t1\n", result.output); JobID jobid = result.job.getID(); TaskAttemptID taskid = new TaskAttemptID(new TaskID(jobid, TaskType.MAP, 1), 0); String userName = UserGroupInformation.getLoginUser().getUserName(); checkTaskDirectories(mr, userName, new String[] { jobid.toString() }, new String[] { taskid.toString() }); // test with maps=0 jobConf = mr.createJobConf(); input = "owen is oom"; result = launchWordCount(jobConf, inDir, outDir, input, 0, 1); assertEquals("is\t1\noom\t1\nowen\t1\n", result.output); Counters counters = result.job.getCounters(); long hdfsRead = counters.findCounter(Task.FILESYSTEM_COUNTER_GROUP, Task.getFileSystemCounterNames("hdfs")[0]).getCounter(); long hdfsWrite = counters.findCounter(Task.FILESYSTEM_COUNTER_GROUP, Task.getFileSystemCounterNames("hdfs")[1]).getCounter(); long rawSplitBytesRead = counters.findCounter(TaskCounter.SPLIT_RAW_BYTES).getCounter(); assertEquals(result.output.length(), hdfsWrite); // add the correction factor of 234 as the input split is also streamed assertEquals(input.length() + rawSplitBytesRead, hdfsRead); // Run a job with input and output going to localfs even though the // default fs is hdfs. { FileSystem localfs = FileSystem.getLocal(jobConf); String TEST_ROOT_DIR = new File(System.getProperty("test.build.data", "/tmp")).toString().replace(' ', '+'); Path localIn = localfs.makeQualified(new Path(TEST_ROOT_DIR + "/local/in")); Path localOut = localfs.makeQualified(new Path(TEST_ROOT_DIR + "/local/out")); result = launchWordCount(jobConf, localIn, localOut, "all your base belong to us", 1, 1); assertEquals("all\t1\nbase\t1\nbelong\t1\nto\t1\nus\t1\nyour\t1\n", result.output); assertTrue("outputs on localfs", localfs.exists(localOut)); }
5526 -1009652200apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None scan all policies once every 5 second SATD_ADDED mySetup(int, int) private void mySetup(int stripeLength, int timeBeforeHar) throws Exception // Make sure data directory exists new File(TEST_DIR).mkdirs(); conf = new Configuration(); conf.set("raid.config.file", CONFIG_FILE); conf.setBoolean("raid.config.reload", true); conf.setLong("raid.config.reload.interval", RELOAD_INTERVAL); // scan all policies once every 5 second conf.setLong("raid.policy.rescan.interval", 5000); // make all deletions not go through Trash conf.set("fs.shell.delete.classname", "org.apache.hadoop.hdfs.DFSClient"); // do not use map-reduce cluster for Raiding conf.set("raid.classname", "org.apache.hadoop.raid.LocalRaidNode"); conf.set("raid.server.address", "localhost:0"); conf.setInt("hdfs.raid.stripeLength", stripeLength); conf.set("hdfs.raid.locations", "/destraid"); dfs = new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DATANODES).build(); dfs.waitActive(); fileSys = dfs.getFileSystem(); namenode = fileSys.getUri().toString(); FileSystem.setDefaultUri(conf, namenode); hftp = "hftp://localhost.localdomain:" + dfs.getNameNodePort(); FileSystem.setDefaultUri(conf, namenode); FileWriter fileWriter = new FileWriter(CONFIG_FILE); fileWriter.write("\n"); String str = " " + " " + " " + "xor " + " /destraid " + " " + "targetReplication " + "1 " + "after RAIDing, decrease the replication factor of a file to this value." + " " + " " + " " + "metaReplication " + "1 " + " replication factor of parity file" + " " + " " + " " + "modTimePeriod " + "2000 " + " time (milliseconds) after a file is modified to make it " + "a candidate for RAIDing " + " " + " "; if (timeBeforeHar >= 0) { str += " " + "time_before_har " + "" + timeBeforeHar + " " + " amount of time waited before har'ing parity files" + " " + " "; } str += "" + "" + ""; fileWriter.write(str); fileWriter.close();
5525 -1009652201apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None TODO: consider more tables and skip tables with non-temp projections SATD_ADDED optimize(Configuration) public static void optimize(Configuration conf) throws Exception VerticaConfiguration vtconfig = new VerticaConfiguration(conf); Connection conn = vtconfig.getConnection(true); // TODO: consider more tables and skip tables with non-temp projections String tableName = vtconfig.getOutputTableName(); Statement stmt = conn.createStatement(); ResultSet rs = null; StringBuffer designTables = new StringBuffer(tableName); HashSet tablesWithTemp = new HashSet(); // fully qualify the table name - defaults to public. if (tableName.indexOf(".") == -1) { tableName = "public." + tableName; } // for now just add the single output table tablesWithTemp.add(tableName); // map from table name to set of projection names HashMap> tableProj = new HashMap>(); rs = stmt.executeQuery("select schemaname, anchortablename, projname from vt_projection;"); while (rs.next()) { String ptable = rs.getString(1) + "." + rs.getString(2); if (!tableProj.containsKey(ptable)) { tableProj.put(ptable, new HashSet()); } tableProj.get(ptable).add(rs.getString(3)); } for (String table : tablesWithTemp) { if (!tableProj.containsKey(table)) { throw new RuntimeException("Cannot optimize table with no data: " + table); } } String designName = (new Integer(conn.hashCode())).toString(); stmt.execute("select create_projection_design('" + designName + "', '', '" + designTables.toString() + "')"); if (VerticaUtil.verticaVersion(conf, true) >= VerticaConfiguration.VERSION_3_5) { stmt.execute("select deploy_design('" + designName + "', '" + designName + "')"); } else { rs = stmt.executeQuery("select get_design_script('" + designName + "', '" + designName + "')"); rs.next(); String[] projSet = rs.getString(1).split(";"); for (String proj : projSet) { stmt.execute(proj); } stmt.execute("select start_refresh()"); // poll for refresh complete boolean refreshing = true; Long timeout = vtconfig.getOptimizePollTimeout(); while (refreshing) { refreshing = false; rs = stmt.executeQuery("select table_name, status from vt_projection_refresh"); while (rs.next()) { String table = rs.getString(1); String stat = rs.getString(2); if (stat.equals("refreshing") && tablesWithTemp.contains(table)) refreshing = true; } rs.close(); Thread.sleep(timeout); } // refresh done, move the ancient history mark (ahm) and drop the temp projections stmt.execute("select make_ahm_now()"); for (String table : tablesWithTemp) { for (String proj : tableProj.get(table)) { stmt.execute("DROP PROJECTION " + proj); } } stmt.close(); }
5524 -1009652202apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None TODO: use explicit date formats SATD_ADDED VerticaRecord> getRecordWriter(TaskAttemptContext) public RecordWriter getRecordWriter(TaskAttemptContext context) throws IOException VerticaConfiguration config = new VerticaConfiguration(context.getConfiguration()); String name = context.getJobName(); // TODO: use explicit date formats String table = config.getOutputTableName(); String copyStmt = "COPY " + table + " FROM STDIN" + " DELIMITER '" + delimiter + "' RECORD TERMINATOR '" + terminator + "' STREAM NAME '" + name + "' DIRECT"; try { Connection conn = config.getConnection(true); return new VerticaRecordWriter(conn, copyStmt, table, delimiter, terminator); } catch (Exception e) { throw new IOException(e); }
5523 -1009652203apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Sufficient to rely on super's implementation SATD_ADDED equals(Object) public boolean equals(Object obj) // Sufficient to rely on super's implementation return (this == obj) || super.equals(obj);
5522 -1009652204apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None TODO: Destroy all the processes in the subtree in this case also. For the time being, killing only the root process. SATD_ADDED setSigKillInterval(long) public void setSigKillInterval(long interval) sleeptimeBeforeSigkill = interval;
5521 -1009652205apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None create files and set permissions 555. Verify if task controller sets the permissions for TT to delete the task dir or work dir properly SATD_ADDED testTaskFilesRemoval(boolean, boolean) private void testTaskFilesRemoval(boolean needCleanup, boolean jvmReuse) throws Exception // Localize job and localize task. TaskTracker.RunningJob rjob = tracker.localizeJob(tip); localizedJobConf = rjob.getJobConf(); if (jvmReuse) { localizedJobConf.setNumTasksToExecutePerJvm(2); } initializeTask(); // TODO: Let the task run and create files. // create files and set permissions 555. Verify if task controller sets // the permissions for TT to delete the task dir or work dir properly validateRemoveTaskFiles(needCleanup, jvmReuse, tip);
5520 -1009652206apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None TODO: Let the task run and create files. SATD_ADDED testTaskFilesRemoval(boolean, boolean) private void testTaskFilesRemoval(boolean needCleanup, boolean jvmReuse) throws Exception // Localize job and localize task. TaskTracker.RunningJob rjob = tracker.localizeJob(tip); localizedJobConf = rjob.getJobConf(); if (jvmReuse) { localizedJobConf.setNumTasksToExecutePerJvm(2); } initializeTask(); // TODO: Let the task run and create files. // create files and set permissions 555. Verify if task controller sets // the permissions for TT to delete the task dir or work dir properly validateRemoveTaskFiles(needCleanup, jvmReuse, tip);
5519 -1009652207apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None work dir should still exist and cleanup queue should be empty SATD_ADDED validateRemoveTaskFiles(boolean, boolean, TaskInProgress) void validateRemoveTaskFiles(boolean needCleanup, boolean jvmReuse, TaskInProgress tip) throws IOException // create files and set permissions 555. Verify if task controller sets // the permissions for TT to delete the taskDir or workDir String dir = (!needCleanup || jvmReuse) ? TaskTracker.getTaskWorkDir(task.getUser(), task.getJobID().toString(), taskId.toString(), task.isTaskCleanupTask()) : TaskTracker.getLocalTaskDir(task.getUser(), task.getJobID().toString(), taskId.toString(), task.isTaskCleanupTask()); Path[] paths = tracker.getLocalFiles(localizedJobConf, dir); assertTrue("No paths found", paths.length > 0); for (Path p : paths) { if (tracker.getLocalFileSystem().exists(p)) { createFileAndSetPermissions(localizedJobConf, p); } } InlineCleanupQueue cleanupQueue = new InlineCleanupQueue(); tracker.setCleanupThread(cleanupQueue); tip.removeTaskFiles(needCleanup, taskId); if (jvmReuse) { // work dir should still exist and cleanup queue should be empty assertTrue("cleanup queue is not empty after removeTaskFiles() in case " + "of jvm reuse.", cleanupQueue.isQueueEmpty()); boolean workDirExists = false; for (Path p : paths) { if (tracker.getLocalFileSystem().exists(p)) { workDirExists = true; } } assertTrue("work dir does not exist in case of jvm reuse", workDirExists); // now try to delete the work dir and verify that there are no stale paths JvmManager.deleteWorkDir(tracker, task); } assertTrue("Some task files are not deleted!! Number of stale paths is " + cleanupQueue.stalePaths.size(), cleanupQueue.stalePaths.size() == 0);
5518 -1009652208apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None If there are a lot of blocks, this returns a random time with in the scan period. Otherwise something sooner. SATD_ADDED hashCode() public int hashCode() return block.hashCode();
5517 -1009652209apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Should we change throttler bandwidth every time bytesLeft changes? not really required. SATD_ADDED hashCode() public int hashCode() return block.hashCode();
5516 -1009652210apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None move map output to reduce input SATD_ADDED getProtocolVersion(String, long) public long getProtocolVersion(String protocol, long clientVersion) return ClientProtocol.versionID;
5515 -1009652211apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Should have incremented the batch count exactly once SATD_ADDED testSyncBatching() public void testSyncBatching() throws Exception // start a cluster Configuration conf = new HdfsConfiguration(); MiniDFSCluster cluster = null; FileSystem fileSys = null; ExecutorService threadA = Executors.newSingleThreadExecutor(); ExecutorService threadB = Executors.newSingleThreadExecutor(); try { cluster = new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DATA_NODES).build(); cluster.waitActive(); fileSys = cluster.getFileSystem(); final FSNamesystem namesystem = cluster.getNamesystem(); FSImage fsimage = namesystem.getFSImage(); final FSEditLog editLog = fsimage.getEditLog(); assertEquals("should start with no txids synced", 0, editLog.getSyncTxId()); // Log an edit from thread A doLogEdit(threadA, editLog, "thread-a 1"); assertEquals("logging edit without syncing should do not affect txid", 0, editLog.getSyncTxId()); // Log an edit from thread B doLogEdit(threadB, editLog, "thread-b 1"); assertEquals("logging edit without syncing should do not affect txid", 0, editLog.getSyncTxId()); // Now ask to sync edit from B, which should sync both edits. doCallLogSync(threadB, editLog); assertEquals("logSync from second thread should bump txid up to 2", 2, editLog.getSyncTxId()); // Now ask to sync edit from A, which was already batched in - thus // it should increment the batch count metric doCallLogSync(threadA, editLog); assertEquals("logSync from first thread shouldn't change txid", 2, editLog.getSyncTxId()); // Should have incremented the batch count exactly once assertCounter("TransactionsBatchedInSync", 1L, getMetrics("NameNodeActivity")); } finally { threadA.shutdown(); threadB.shutdown(); if (fileSys != null) fileSys.close(); if (cluster != null) cluster.shutdown(); }
5514 -1009652212apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Similar to BlockReport_08 but corrupts GS and len of the TEMPORARY's replica block. Expect the same behaviour: NN should simply ignore this block SATD_ADDED startUpCluster() public void startUpCluster() throws IOException // Reset if case a test has modified the value REPL_FACTOR = 1; cluster = new MiniDFSCluster.Builder(conf).numDataNodes(REPL_FACTOR).build(); fs = (DistributedFileSystem) cluster.getFileSystem(); bpid = cluster.getNamesystem().getBlockPoolId();
5513 -1009652213apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None TODO change the default with the actual stats TODO do u need varied sized words? SATD_ADDED getRandomTextDataGeneratorListSize(Configuration) static int getRandomTextDataGeneratorListSize(Configuration conf) return conf.getInt(GRIDMIX_DATAGEN_RANDOMTEXT_LISTSIZE, DEFAULT_LIST_SIZE);
5512 -1009652214apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None TODO: Need to come up with a better place to put this block of code to do with reading the file SATD_ADDED getSecretKey(Credentials, Text) public static byte[] getSecretKey(Credentials credentials, Text alias) if (credentials == null) return null; return credentials.getSecretKey(alias);
5511 -1009652215apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None now take care of the rest of the files and subdirectories SATD_ADDED linkBlocks(File, File, int, HardLink) static void linkBlocks(File from, File to, int oldLV, HardLink hl) throws IOException if (!from.exists()) { return; } if (!from.isDirectory()) { if (from.getName().startsWith(COPY_FILE_PREFIX)) { FileInputStream in = new FileInputStream(from); try { FileOutputStream out = new FileOutputStream(to); try { IOUtils.copyBytes(in, out, 16 * 1024); hl.linkStats.countPhysicalFileCopies++; } finally { out.close(); } } finally { in.close(); } } else { // check if we are upgrading from pre-generation stamp version. if (oldLV >= PRE_GENERATIONSTAMP_LAYOUT_VERSION) { // Link to the new file name. to = new File(convertMetatadataFileName(to.getAbsolutePath())); } HardLink.createHardLink(from, to); hl.linkStats.countSingleLinks++; } return; } // from is a directory hl.linkStats.countDirs++; if (!to.mkdirs()) throw new IOException("Cannot create directory " + to); // If upgrading from old stuff, need to munge the filenames. That has to // be done one file at a time, so hardlink them one at a time (slow). if (oldLV >= PRE_GENERATIONSTAMP_LAYOUT_VERSION) { String[] blockNames = from.list(new java.io.FilenameFilter() { public boolean accept(File dir, String name) { return name.startsWith(BLOCK_SUBDIR_PREFIX) || name.startsWith(BLOCK_FILE_PREFIX) || name.startsWith(COPY_FILE_PREFIX); } }); if (blockNames.length == 0) { hl.linkStats.countEmptyDirs++; } else for (int i = 0; i < blockNames.length; i++) linkBlocks(new File(from, blockNames[i]), new File(to, blockNames[i]), oldLV, hl); } else { // If upgrading from a relatively new version, we only need to create // links with the same filename. This can be done in bulk (much faster). String[] blockNames = from.list(new java.io.FilenameFilter() { public boolean accept(File dir, String name) { return name.startsWith(BLOCK_FILE_PREFIX); } }); if (blockNames.length > 0) { HardLink.createHardLinkMult(from, blockNames, to); hl.linkStats.countMultLinks++; hl.linkStats.countFilesMultLinks += blockNames.length; } else { hl.linkStats.countEmptyDirs++; } // now take care of the rest of the files and subdirectories String[] otherNames = from.list(new java.io.FilenameFilter() { public boolean accept(File dir, String name) { return name.startsWith(BLOCK_SUBDIR_PREFIX) || name.startsWith(COPY_FILE_PREFIX); } }); for (int i = 0; i < otherNames.length; i++) linkBlocks(new File(from, otherNames[i]), new File(to, otherNames[i]), oldLV, hl); }
5510 -1009652216apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None TODO Use StringBuilder instead SATD_ADDED toString() public String toString() // TODO Use StringBuilder instead return " Overloaded = " + overloaded() + ", MapSlotBackfill = " + mapSlotsBackfill + ", MapSlotCapacity = " + mapSlotCapacity + ", ReduceSlotBackfill = " + reduceSlotsBackfill + ", ReduceSlotCapacity = " + reduceSlotCapacity + ", NumJobsBackfill = " + numJobsBackfill;
5509 -1009652217apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None TODO: We need to take care of scenario when one map/reduce takes more than 1 slot. SATD_ADDED createReaderThread() public Thread createReaderThread() return new StressReaderThread("StressJobFactory");
5508 -1009652218apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None pcnt would fail resetSpill doesn't move bufindex, kvindex SATD_ADDED isMapTask() public boolean isMapTask() return true;
5506 -1009652220apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None todo: only if all classes are of proper types SATD_ADDED isValid(ILaunchConfiguration) public boolean isValid(ILaunchConfiguration launchConfig) // todo: only if all classes are of proper types return true;
5505 -1009652221apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None scan all policies once every 5 second SATD_ADDED createClusters(boolean) private void createClusters(boolean local) throws Exception // Make sure data directory exists new File(TEST_DIR).mkdirs(); conf = new Configuration(); conf.set("raid.config.file", CONFIG_FILE); conf.setBoolean("raid.config.reload", true); conf.setLong("raid.config.reload.interval", RELOAD_INTERVAL); // scan all policies once every 5 second conf.setLong("raid.policy.rescan.interval", 5000); // make all deletions not go through Trash conf.set("fs.shell.delete.classname", "org.apache.hadoop.dfs.DFSClient"); // the RaidNode does the raiding inline (instead of submitting to map/reduce) if (local) { conf.set("raid.classname", "org.apache.hadoop.raid.LocalRaidNode"); } else { conf.set("raid.classname", "org.apache.hadoop.raid.DistRaidNode"); } conf.set("raid.server.address", "localhost:0"); // create a dfs and map-reduce cluster final int taskTrackers = 4; final int jobTrackerPort = 60050; dfs = new MiniDFSCluster(conf, 3, true, null); dfs.waitActive(); fileSys = dfs.getFileSystem(); namenode = fileSys.getUri().toString(); mr = new MiniMRCluster(taskTrackers, namenode, 3); jobTrackerName = "localhost:" + mr.getJobTrackerPort(); hftp = "hftp://localhost.localdomain:" + dfs.getNameNodePort(); FileSystem.setDefaultUri(conf, namenode); conf.set("mapred.job.tracker", jobTrackerName);
5504 -1009652222apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None TODO: We should first fix the files that lose more blocks SATD_ADDED sortCorruptFiles(List) void sortCorruptFiles(List files) // TODO: We should first fix the files that lose more blocks Comparator comp = new Comparator() { public int compare(Path p1, Path p2) { if (isXorParityFile(p2) || isRsParityFile(p2)) { // If p2 is a parity file, p1 is smaller. return -1; } if (isXorParityFile(p1) || isRsParityFile(p1)) { // If p1 is a parity file, p2 is smaller. return 1; } // If both are source files, they are equal. return 0; } }; Collections.sort(files, comp);
5503 -1009652223apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None TODO: use explicit date formats SATD_ADDED Text> getRecordWriter(TaskAttemptContext) public RecordWriter getRecordWriter(TaskAttemptContext context) throws IOException Configuration conf = context.getConfiguration(); VerticaConfiguration vtconfig = new VerticaConfiguration(conf); String name = context.getJobName(); delimiter = vtconfig.getOutputDelimiter(); terminator = vtconfig.getOutputRecordTerminator(); // TODO: use explicit date formats String table = vtconfig.getOutputTableName(); String copyStmt = "COPY " + table + " FROM STDIN" + " DELIMITER '" + delimiter + "' RECORD TERMINATOR '" + terminator + "' STREAM NAME '" + name + "' DIRECT"; try { Connection conn = vtconfig.getConnection(true); return new VerticaStreamingRecordWriter(conn, copyStmt, table); } catch (Exception e) { throw new IOException(e); }
5502 -1009652224apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None BlockInfo BlockInfoUnderConstruction participates in maps the same way as BlockInfo SATD_ADDED hashCode() public int hashCode() return super.hashCode();
5501 -1009652225apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Sufficient to rely on super's implementation SATD_ADDED equals(Object) public boolean equals(Object obj) // Sufficient to rely on super's implementation return (this == obj) || super.equals(obj);
5500 -1009652226apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None domain name too short too long too large weird too short too long too large too large SATD_ADDED testIsIPAddress() public void testIsIPAddress() final String[] positives = { // regular ipv4 "123.13.42.255", // padded 0 "123.01.0.255", // more padded 0 "000.001.002.020", // escaped . "123\\.13\\.42\\.255", // all-zero "0.0.0.0", // all-0xff "255.255.255.255", // regular ipv6 "1080:0:0:0:8:800:200C:417A", // padded 0 "1080:01:020:3:8:0800:200C:417A", // more padded 0 "1080:01:002:0003:080:0800:0200:417A", // all-zero "0:0:0:0:0:0:0:0", // all-0xff "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff" }; final String[] negatives = { // domain name "node.megatron.com", // too short "13.42.255", // too long "123.13.42.255.10", // too large "123.256.42.255", // weird "123.13.42.255.weird.com", // too short "1080:0:0:0:8:200C:417A", // too long "1080:0:0:0:1:8:800:200C:417A", // too large "1080A:0:0:0:8:800:200C:417A", // too large "1080:0:0:0:8:800:200G:417A" }; for (String s : positives) { Assert.assertTrue(s, SimulatorEngine.isIPAddress(s)); } for (String s : negatives) { Assert.assertFalse(s, SimulatorEngine.isIPAddress(s)); }
5499 -1009652227apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None scan all policies once every 5 second SATD_ADDED createClusters(boolean) private void createClusters(boolean local) throws Exception // Make sure data directory exists new File(TEST_DIR).mkdirs(); conf = new Configuration(); conf.set("raid.config.file", CONFIG_FILE); conf.setBoolean("raid.config.reload", true); conf.setLong("raid.config.reload.interval", RELOAD_INTERVAL); // scan all policies once every 5 second conf.setLong("raid.policy.rescan.interval", 5000); // make all deletions not go through Trash conf.set("fs.shell.delete.classname", "org.apache.hadoop.hdfs.DFSClient"); // the RaidNode does the raiding inline (instead of submitting to map/reduce) if (local) { conf.set("raid.classname", "org.apache.hadoop.raid.LocalRaidNode"); } else { conf.set("raid.classname", "org.apache.hadoop.raid.DistRaidNode"); } conf.set("raid.server.address", "localhost:0"); conf.set(RaidNode.RAID_LOCATION_KEY, "/destraid"); // create a dfs and map-reduce cluster final int taskTrackers = 4; dfs = new MiniDFSCluster(conf, 3, true, null); dfs.waitActive(); fileSys = dfs.getFileSystem(); namenode = fileSys.getUri().toString(); mr = new MiniMRCluster(taskTrackers, namenode, 3); jobTrackerName = "localhost:" + mr.getJobTrackerPort(); hftp = "hftp://localhost.localdomain:" + dfs.getNameNodePort(); FileSystem.setDefaultUri(conf, namenode); conf.set("mapred.job.tracker", jobTrackerName);
5498 -1009652228apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None move the blocks SATD_ADDED testConcatNotCompleteBlock() public void testConcatNotCompleteBlock() throws IOException long trgFileLen = blockSize * 3; // block at the end - not full long srcFileLen = blockSize * 3 + 20; // create first file String name1 = "/trg", name2 = "/src"; Path filePath1 = new Path(name1); DFSTestUtil.createFile(dfs, filePath1, trgFileLen, REPL_FACTOR, 1); HdfsFileStatus fStatus = cluster.getNameNode().getFileInfo(name1); long fileLen = fStatus.getLen(); assertEquals(fileLen, trgFileLen); // read the file FSDataInputStream stm = dfs.open(filePath1); byte[] byteFile1 = new byte[(int) trgFileLen]; stm.readFully(0, byteFile1); stm.close(); LocatedBlocks lb1 = cluster.getNameNode().getBlockLocations(name1, 0, trgFileLen); Path filePath2 = new Path(name2); DFSTestUtil.createFile(dfs, filePath2, srcFileLen, REPL_FACTOR, 1); fStatus = cluster.getNameNode().getFileInfo(name2); fileLen = fStatus.getLen(); assertEquals(srcFileLen, fileLen); // read the file stm = dfs.open(filePath2); byte[] byteFile2 = new byte[(int) srcFileLen]; stm.readFully(0, byteFile2); stm.close(); LocatedBlocks lb2 = cluster.getNameNode().getBlockLocations(name2, 0, srcFileLen); System.out.println("trg len=" + trgFileLen + "; src len=" + srcFileLen); // move the blocks dfs.concat(filePath1, new Path[] { filePath2 }); long totalLen = trgFileLen + srcFileLen; fStatus = cluster.getNameNode().getFileInfo(name1); fileLen = fStatus.getLen(); // read the resulting file stm = dfs.open(filePath1); byte[] byteFileConcat = new byte[(int) fileLen]; stm.readFully(0, byteFileConcat); stm.close(); LocatedBlocks lbConcat = cluster.getNameNode().getBlockLocations(name1, 0, fileLen); // verifications // 1. number of blocks assertEquals(lbConcat.locatedBlockCount(), lb1.locatedBlockCount() + lb2.locatedBlockCount()); // 2. file lengths System.out.println("file1 len=" + fileLen + "; total len=" + totalLen); assertEquals(fileLen, totalLen); // 3. removal of the src file fStatus = cluster.getNameNode().getFileInfo(name2); // file shouldn't exist assertNull("File " + name2 + "still exists", fStatus); // 4. content checkFileContent(byteFileConcat, new byte[][] { byteFile1, byteFile2 });
5497 -1009652229apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Make sure we really waited for the flush to complete! SATD_ADDED testSaveRightBeforeSync() public void testSaveRightBeforeSync() throws Exception Configuration conf = getConf(); NameNode.initMetrics(conf, NamenodeRole.ACTIVE); DFSTestUtil.formatNameNode(conf); final FSNamesystem namesystem = new FSNamesystem(conf); try { FSImage fsimage = namesystem.getFSImage(); FSEditLog editLog = spy(fsimage.getEditLog()); fsimage.editLog = editLog; final AtomicReference deferredException = new AtomicReference(); final CountDownLatch waitToEnterSync = new CountDownLatch(1); final Thread doAnEditThread = new Thread() { public void run() { try { LOG.info("Starting mkdirs"); namesystem.mkdirs("/test", new PermissionStatus("test", "test", new FsPermission((short) 00755)), true); LOG.info("mkdirs complete"); } catch (Throwable ioe) { deferredException.set(ioe); waitToEnterSync.countDown(); } } }; Answer blockingSync = new Answer() { @Override public Void answer(InvocationOnMock invocation) throws Throwable { LOG.info("logSync called"); if (Thread.currentThread() == doAnEditThread) { LOG.info("edit thread: Telling main thread we made it just before logSync..."); waitToEnterSync.countDown(); LOG.info("edit thread: sleeping for " + BLOCK_TIME + "secs"); Thread.sleep(BLOCK_TIME * 1000); LOG.info("Going through to logSync. This will allow the main thread to continue."); } invocation.callRealMethod(); LOG.info("logSync complete"); return null; } }; doAnswer(blockingSync).when(editLog).logSync(); doAnEditThread.start(); LOG.info("Main thread: waiting to just before logSync..."); waitToEnterSync.await(); assertNull(deferredException.get()); LOG.info("Main thread: detected that logSync about to be called."); LOG.info("Trying to enter safe mode."); LOG.info("This should block for " + BLOCK_TIME + "sec, since we have pending edits"); long st = System.currentTimeMillis(); namesystem.setSafeMode(SafeModeAction.SAFEMODE_ENTER); long et = System.currentTimeMillis(); LOG.info("Entered safe mode"); // Make sure we really waited for the flush to complete! assertTrue(et - st > (BLOCK_TIME - 1) * 1000); // Once we're in safe mode, save namespace. namesystem.saveNamespace(); LOG.info("Joining on edit thread..."); doAnEditThread.join(); assertNull(deferredException.get()); verifyEditLogs(namesystem, fsimage); } finally { LOG.info("Closing namesystem"); if (namesystem != null) namesystem.close(); }
5496 -1009652230apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Make sure we really waited for the flush to complete! SATD_ADDED testSaveImageWhileSyncInProgress() public void testSaveImageWhileSyncInProgress() throws Exception Configuration conf = getConf(); NameNode.initMetrics(conf, NamenodeRole.ACTIVE); DFSTestUtil.formatNameNode(conf); final FSNamesystem namesystem = new FSNamesystem(conf); try { FSImage fsimage = namesystem.getFSImage(); FSEditLog editLog = fsimage.getEditLog(); ArrayList streams = editLog.getEditStreams(); EditLogOutputStream spyElos = spy(streams.get(0)); streams.set(0, spyElos); final AtomicReference deferredException = new AtomicReference(); final CountDownLatch waitToEnterFlush = new CountDownLatch(1); final Thread doAnEditThread = new Thread() { public void run() { try { LOG.info("Starting mkdirs"); namesystem.mkdirs("/test", new PermissionStatus("test", "test", new FsPermission((short) 00755)), true); LOG.info("mkdirs complete"); } catch (Throwable ioe) { deferredException.set(ioe); waitToEnterFlush.countDown(); } } }; Answer blockingFlush = new Answer() { @Override public Void answer(InvocationOnMock invocation) throws Throwable { LOG.info("Flush called"); if (Thread.currentThread() == doAnEditThread) { LOG.info("edit thread: Telling main thread we made it to flush section..."); // Signal to main thread that the edit thread is in the racy section waitToEnterFlush.countDown(); LOG.info("edit thread: sleeping for " + BLOCK_TIME + "secs"); Thread.sleep(BLOCK_TIME * 1000); LOG.info("Going through to flush. This will allow the main thread to continue."); } invocation.callRealMethod(); LOG.info("Flush complete"); return null; } }; doAnswer(blockingFlush).when(spyElos).flush(); doAnEditThread.start(); // Wait for the edit thread to get to the logsync unsynchronized section LOG.info("Main thread: waiting to enter flush..."); waitToEnterFlush.await(); assertNull(deferredException.get()); LOG.info("Main thread: detected that logSync is in unsynchronized section."); LOG.info("Trying to enter safe mode."); LOG.info("This should block for " + BLOCK_TIME + "sec, since flush will sleep that long"); long st = System.currentTimeMillis(); namesystem.setSafeMode(SafeModeAction.SAFEMODE_ENTER); long et = System.currentTimeMillis(); LOG.info("Entered safe mode"); // Make sure we really waited for the flush to complete! assertTrue(et - st > (BLOCK_TIME - 1) * 1000); // Once we're in safe mode, save namespace. namesystem.saveNamespace(); LOG.info("Joining on edit thread..."); doAnEditThread.join(); assertNull(deferredException.get()); verifyEditLogs(namesystem, fsimage); } finally { LOG.info("Closing namesystem"); if (namesystem != null) namesystem.close(); }
5495 -1009652231apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None verify that 'format' really blew away all pre-existing files SATD_ADDED testCheckpoint(StartupOption) void testCheckpoint(StartupOption op) throws IOException Path file1 = new Path("checkpoint.dat"); Path file2 = new Path("checkpoint2.dat"); Configuration conf = new HdfsConfiguration(); short replication = (short) conf.getInt(DFSConfigKeys.DFS_REPLICATION_KEY, 3); conf.set(DFSConfigKeys.DFS_BLOCKREPORT_INITIAL_DELAY_KEY, "0"); // disable block scanner conf.setInt(DFSConfigKeys.DFS_DATANODE_SCAN_PERIOD_HOURS_KEY, -1); int numDatanodes = Math.max(3, replication); MiniDFSCluster cluster = null; FileSystem fileSys = null; BackupNode backup = null; try { cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDatanodes).build(); fileSys = cluster.getFileSystem(); // // verify that 'format' really blew away all pre-existing files // assertTrue(!fileSys.exists(file1)); assertTrue(!fileSys.exists(file2)); // // Create file1 // writeFile(fileSys, file1, replication); checkFile(fileSys, file1, replication); // // Take a checkpoint // backup = startBackupNode(conf, op, 1); waitCheckpointDone(backup); } catch (IOException e) { LOG.error("Error in TestBackupNode:", e); assertTrue(e.getLocalizedMessage(), false); } finally { if (backup != null) backup.stop(); if (fileSys != null) fileSys.close(); if (cluster != null) cluster.shutdown(); } File imageFileNN = new File(BASE_DIR, "name1/current/fsimage"); File imageFileBN = new File(getBackupNodeDir(op, 1), "/current/fsimage"); LOG.info("NameNode fsimage length = " + imageFileNN.length()); LOG.info("Backup Node fsimage length = " + imageFileBN.length()); assertTrue(imageFileNN.length() == imageFileBN.length()); try { // // Restart cluster and verify that file1 still exist. // cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDatanodes).format(false).build(); fileSys = cluster.getFileSystem(); // check that file1 still exists checkFile(fileSys, file1, replication); cleanupFile(fileSys, file1); // create new file file2 writeFile(fileSys, file2, replication); checkFile(fileSys, file2, replication); // // Take a checkpoint // backup = startBackupNode(conf, op, 1); waitCheckpointDone(backup); } catch (IOException e) { LOG.error("Error in TestBackupNode:", e); assertTrue(e.getLocalizedMessage(), false); } finally { if (backup != null) backup.stop(); if (fileSys != null) fileSys.close(); if (cluster != null) cluster.shutdown(); } LOG.info("NameNode fsimage length = " + imageFileNN.length()); LOG.info("Backup Node fsimage length = " + imageFileBN.length()); assertTrue(imageFileNN.length() == imageFileBN.length()); try { // // Restart cluster and verify that file2 exists and // file1 does not exist. // cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDatanodes).format(false).build(); fileSys = cluster.getFileSystem(); assertTrue(!fileSys.exists(file1)); // verify that file2 exists checkFile(fileSys, file2, replication); } catch (IOException e) { LOG.error("Error in TestBackupNode:", e); assertTrue(e.getLocalizedMessage(), false); } finally { fileSys.close(); cluster.shutdown(); }
5494 -1009652232apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None TODO: Fix SATD_ADDED dump() private synchronized void dump() synchronized (eventLog) { eventLog.log("BEGIN_DUMP"); // List jobs in order of submit time ArrayList jobs = new ArrayList(infos.keySet()); Collections.sort(jobs, new Comparator() { public int compare(JobInProgress j1, JobInProgress j2) { return (int) Math.signum(j1.getStartTime() - j2.getStartTime()); } }); // Dump info for each job for (JobInProgress job : jobs) { JobProfile profile = job.getProfile(); JobInfo info = infos.get(job); Schedulable ms = info.mapSchedulable; Schedulable rs = info.reduceSchedulable; eventLog.log("JOB", profile.getJobID(), profile.name, profile.user, job.getPriority(), poolMgr.getPoolName(job), job.numMapTasks, ms.getRunningTasks(), ms.getDemand(), ms.getFairShare(), ms.getWeight(), job.numReduceTasks, rs.getRunningTasks(), rs.getDemand(), rs.getFairShare(), rs.getWeight()); } // List pools in alphabetical order List pools = new ArrayList(poolMgr.getPools()); Collections.sort(pools, new Comparator() { public int compare(Pool p1, Pool p2) { if (p1.isDefaultPool()) return 1; else if (p2.isDefaultPool()) return -1; else return p1.getName().compareTo(p2.getName()); } }); for (Pool pool : pools) { int runningMaps = 0; int runningReduces = 0; for (JobInProgress job : pool.getJobs()) { JobInfo info = infos.get(job); if (info != null) { // TODO: Fix // runningMaps += info.runningMaps; // runningReduces += info.runningReduces; } } String name = pool.getName(); eventLog.log("POOL", name, poolMgr.getPoolWeight(name), pool.getJobs().size(), poolMgr.getAllocation(name, TaskType.MAP), runningMaps, poolMgr.getAllocation(name, TaskType.REDUCE), runningReduces); } // Dump info for each pool eventLog.log("END_DUMP"); }
5493 -1009652233apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Compute total map/reduce slots In the future we can precompute this if the Scheduler becomes a listener of tracker join/leave events. SATD_ADDED assignTasks(TaskTracker) public synchronized List assignTasks(TaskTracker tracker) throws IOException if (// Don't try to assign tasks if we haven't yet started up !initialized) return null; String trackerName = tracker.getTrackerName(); eventLog.log("HEARTBEAT", trackerName); long currentTime = clock.getTime(); // Compute total runnable maps and reduces, and currently running ones int runnableMaps = 0; int runningMaps = 0; int runnableReduces = 0; int runningReduces = 0; for (Pool pool : poolMgr.getPools()) { runnableMaps += pool.getMapSchedulable().getDemand(); runningMaps += pool.getMapSchedulable().getRunningTasks(); runnableReduces += pool.getReduceSchedulable().getDemand(); runningReduces += pool.getReduceSchedulable().getRunningTasks(); } ClusterStatus clusterStatus = taskTrackerManager.getClusterStatus(); // Compute total map/reduce slots // In the future we can precompute this if the Scheduler becomes a // listener of tracker join/leave events. int totalMapSlots = getTotalSlots(TaskType.MAP, clusterStatus); int totalReduceSlots = getTotalSlots(TaskType.REDUCE, clusterStatus); eventLog.log("RUNNABLE_TASKS", runnableMaps, runningMaps, runnableReduces, runningReduces); // Update time waited for local maps for jobs skipped on last heartbeat updateLocalityWaitTimes(currentTime); TaskTrackerStatus tts = tracker.getStatus(); // loop counter for map in the below while loop int mapsAssigned = 0; // loop counter for reduce in the below while int reducesAssigned = 0; int mapCapacity = maxTasksToAssign(TaskType.MAP, tts); int reduceCapacity = maxTasksToAssign(TaskType.REDUCE, tts); // flag used for ending the loop boolean mapRejected = false; // flag used for ending the loop boolean reduceRejected = false; // Keep track of which jobs were visited for map tasks and which had tasks // launched, so that we can later mark skipped jobs for delay scheduling Set visitedForMap = new HashSet(); Set visitedForReduce = new HashSet(); Set launchedMap = new HashSet(); ArrayList tasks = new ArrayList(); // Scan jobs to assign tasks until neither maps nor reduces can be assigned while (true) { // Computing the ending conditions for the loop // Reject a task type if one of the following condition happens // 1. number of assigned task reaches per heatbeat limit // 2. number of running tasks reaches runnable tasks // 3. task is rejected by the LoadManager.canAssign if (!mapRejected) { if (mapsAssigned == mapCapacity || runningMaps == runnableMaps || !loadMgr.canAssignMap(tts, runnableMaps, totalMapSlots)) { eventLog.log("INFO", "Can't assign another MAP to " + trackerName); mapRejected = true; } } if (!reduceRejected) { if (reducesAssigned == reduceCapacity || runningReduces == runnableReduces || !loadMgr.canAssignReduce(tts, runnableReduces, totalReduceSlots)) { eventLog.log("INFO", "Can't assign another REDUCE to " + trackerName); reduceRejected = true; } } // Exit while (true) loop if // 1. neither maps nor reduces can be assigned // 2. assignMultiple is off and we already assigned one task if (mapRejected && reduceRejected || !assignMultiple && tasks.size() > 0) { // This is the only exit of the while (true) loop break; } // Determine which task type to assign this time // First try choosing a task type which is not rejected TaskType taskType; if (mapRejected) { taskType = TaskType.REDUCE; } else if (reduceRejected) { taskType = TaskType.MAP; } else { // If both types are available, choose the task type with fewer running // tasks on the task tracker to prevent that task type from starving if (tts.countMapTasks() <= tts.countReduceTasks()) { taskType = TaskType.MAP; } else { taskType = TaskType.REDUCE; } } // Get the map or reduce schedulables and sort them by fair sharing List scheds = getPoolSchedulables(taskType); Collections.sort(scheds, new SchedulingAlgorithms.FairShareComparator()); boolean foundTask = false; for (Schedulable sched : scheds) { // This loop will assign only one task eventLog.log("INFO", "Checking for " + taskType + " task in " + sched.getName()); Task task = taskType == TaskType.MAP ? sched.assignTask(tts, currentTime, visitedForMap) : sched.assignTask(tts, currentTime, visitedForReduce); if (task != null) { foundTask = true; JobInProgress job = taskTrackerManager.getJob(task.getJobID()); eventLog.log("ASSIGN", trackerName, taskType, job.getJobID(), task.getTaskID()); // Update running task counts, and the job's locality level if (taskType == TaskType.MAP) { launchedMap.add(job); mapsAssigned++; runningMaps++; updateLastMapLocalityLevel(job, task, tts); } else { reducesAssigned++; runningReduces++; } // Add task to the list of assignments tasks.add(task); // This break makes this loop assign only one task break; } // end if(task != null) } // end for(Schedulable sched: scheds) // Reject the task type if we cannot find a task if (!foundTask) { if (taskType == TaskType.MAP) { mapRejected = true; } else { reduceRejected = true; } } } // end while (true) // Mark any jobs that were visited for map tasks but did not launch a task // as skipped on this heartbeat for (JobInProgress job : visitedForMap) { if (!launchedMap.contains(job)) { infos.get(job).skippedAtLastHeartbeat = true; } } // If no tasks were found, return null return tasks.isEmpty() ? null : tasks;
5490 -1009652236apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None start time for virtual clock possible improvement: set default value to sth more meaningful based on the 1st job SATD_ADDED startTaskTrackers(ClusterStory, JobConf, long) /** * port assigned to TTs, incremented by 1 for each TT */ int port = 10000; int numTaskTrackers = 0; Random random = new Random(RandomSeedGenerator.getSeed("forStartTaskTrackers()", masterRandomSeed)); final int startDuration = jobConf.getInt("mumak.cluster.startup.duration", DEFAULT_CLUSTER_STARTUP_DURATION); for (MachineNode node : cluster.getMachines()) { jobConf.set("mumak.tasktracker.host.name", node.getName()); jobConf.set("mumak.tasktracker.tracker.name", "tracker_" + node.getName() + ":localhost/127.0.0.1:" + port); long subRandomSeed = RandomSeedGenerator.getSeed("forTaskTracker" + numTaskTrackers, masterRandomSeed); jobConf.setLong("mumak.tasktracker.random.seed", subRandomSeed); numTaskTrackers++; port++; SimulatorTaskTracker tt = new SimulatorTaskTracker(jt, jobConf); long firstHeartbeat = now + random.nextInt(startDuration); queue.addAll(tt.init(firstHeartbeat)); } // In startDuration + heartbeat interval of the full cluster time each // TT is started up and told on its 2nd heartbeat to beat at a rate // corresponding to the steady state of the cluster long clusterSteady = now + startDuration + jt.getNextHeartbeatInterval(); return clusterSteady; long startTaskTrackers(ClusterStory cluster, JobConf jobConf, long now)
5489 -1009652237apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None possible improvement: change date format to human readable ? SATD_ADDED startTaskTrackers(ClusterStory, JobConf, long) /** * port assigned to TTs, incremented by 1 for each TT */ int port = 10000; int numTaskTrackers = 0; Random random = new Random(RandomSeedGenerator.getSeed("forStartTaskTrackers()", masterRandomSeed)); final int startDuration = jobConf.getInt("mumak.cluster.startup.duration", DEFAULT_CLUSTER_STARTUP_DURATION); for (MachineNode node : cluster.getMachines()) { jobConf.set("mumak.tasktracker.host.name", node.getName()); jobConf.set("mumak.tasktracker.tracker.name", "tracker_" + node.getName() + ":localhost/127.0.0.1:" + port); long subRandomSeed = RandomSeedGenerator.getSeed("forTaskTracker" + numTaskTrackers, masterRandomSeed); jobConf.setLong("mumak.tasktracker.random.seed", subRandomSeed); numTaskTrackers++; port++; SimulatorTaskTracker tt = new SimulatorTaskTracker(jt, jobConf); long firstHeartbeat = now + random.nextInt(startDuration); queue.addAll(tt.init(firstHeartbeat)); } // In startDuration + heartbeat interval of the full cluster time each // TT is started up and told on its 2nd heartbeat to beat at a rate // corresponding to the steady state of the cluster long clusterSteady = now + startDuration + jt.getNextHeartbeatInterval(); return clusterSteady; long startTaskTrackers(ClusterStory cluster, JobConf jobConf, long now)
5488 -1009652238apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None TODO cache SATD_ADDED getInputFiles(long, Collection) public long getInputFiles(long minSize, Collection files) throws IOException updateLock.readLock().lock(); try { return root.selectFiles(minSize, files); } finally { updateLock.readLock().unlock(); }
5487 -1009652239apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Exclude this task from next search because it is already considered. SATD_ADDED killTasksWithLeastProgress(long) private void killTasksWithLeastProgress(long memoryStillInUsage) List tasksToKill = new ArrayList(); List tasksToExclude = new ArrayList(); // Find tasks to kill so as to get memory usage under limits. while (memoryStillInUsage > maxMemoryAllowedForAllTasks) { // Exclude tasks that are already marked for killing. // Note that we do not need to call isKillable() here because the logic // is contained in taskTracker.findTaskToKill() TaskInProgress task = taskTracker.findTaskToKill(tasksToExclude); if (task == null) { // couldn't find any more tasks to kill. break; } TaskAttemptID tid = task.getTask().getTaskID(); if (processTreeInfoMap.containsKey(tid)) { ProcessTreeInfo ptInfo = processTreeInfoMap.get(tid); ProcfsBasedProcessTree pTree = ptInfo.getProcessTree(); memoryStillInUsage -= pTree.getCumulativeVmem(); tasksToKill.add(tid); } // Exclude this task from next search because it is already // considered. tasksToExclude.add(tid); } // Now kill the tasks. if (!tasksToKill.isEmpty()) { for (TaskAttemptID tid : tasksToKill) { String msg = "Killing one of the least progress tasks - " + tid + ", as the cumulative memory usage of all the tasks on " + "the TaskTracker exceeds virtual memory limit " + maxMemoryAllowedForAllTasks + "."; LOG.warn(msg); killTask(tid, msg); } } else { LOG.info("The total memory usage is overflowing TTs limits. " + "But found no alive task to kill for freeing memory."); }
5486 -1009652240apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None TODO What is compressible is turned on? LOG is a bad idea! SATD_ADDED write(DataOutput) public void write(DataOutput out) throws IOException // data bytes including vint encoding WritableUtils.writeVInt(out, size); final int payload = size - WritableUtils.getVIntSize(size); if (payload > Long.SIZE / Byte.SIZE) { if (compressible) { writeRandomText(out, payload); } else { writeRandom(out, payload); } } else if (payload > 0) { // TODO What is compressible is turned on? LOG is a bad idea! out.write(literal, 0, payload); }
5485 -1009652241apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None TODO Should we use long for size. What if the data is more than 4G? SATD_ADDED writeRandomText(DataOutput, int) private void writeRandomText(DataOutput out, final int size) throws IOException long tmp = seed; out.writeLong(tmp); int i = size - (Long.SIZE / Byte.SIZE); // TODO Should we use long for size. What if the data is more than 4G? String randomWord = rtg.getRandomWord(); byte[] bytes = randomWord.getBytes("UTF-8"); long randomWordSize = bytes.length; while (i >= randomWordSize) { out.write(bytes); i -= randomWordSize; // get the next random word randomWord = rtg.getRandomWord(); bytes = randomWord.getBytes("UTF-8"); // determine the random word size randomWordSize = bytes.length; } // pad the remaining bytes if (i > 0) { out.write(bytes, 0, i); }
5484 -1009652242apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None TODO: Destroy all the processes in the subtree in this case also. For the time being, killing only the root process. SATD_ADDED isSetsidSupported() private static boolean isSetsidSupported() ShellCommandExecutor shexec = null; boolean setsidSupported = true; try { String[] args = { "setsid", "bash", "-c", "echo $$" }; shexec = new ShellCommandExecutor(args); shexec.execute(); } catch (IOException ioe) { LOG.warn("setsid is not available on this machine. So not using it."); setsidSupported = false; } finally { // handle the exit code LOG.info("setsid exited with exit code " + shexec.getExitCode()); } return setsidSupported;
5483 -1009652243apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None turn down the completion poll interval from the 5 second default for better test performance. SATD_ADDED testWithDFS() public void testWithDFS() throws IOException MiniDFSCluster dfs = null; MiniMRCluster mr = null; FileSystem fileSys = null; try { final int taskTrackers = 4; Configuration conf = new Configuration(); dfs = new MiniDFSCluster(conf, 4, true, null); fileSys = dfs.getFileSystem(); mr = new MiniMRCluster(taskTrackers, fileSys.getUri().toString(), 1); JobTracker jt = mr.getJobTrackerRunner().getJobTracker(); final Path inDir = new Path("./input"); final Path outDir = new Path("./output"); String input = "The quick brown fox\nhas many silly\nred fox sox\n"; // launch job with fail tasks JobConf jobConf = mr.createJobConf(); // turn down the completion poll interval from the 5 second default // for better test performance. jobConf.set(Job.COMPLETION_POLL_INTERVAL_KEY, "50"); jobConf.setOutputCommitter(CommitterWithLogs.class); RunningJob rJob = launchJob(jobConf, inDir, outDir, input); rJob.waitForCompletion(); validateJob(rJob, jt, true); // launch job with fail tasks and fail-cleanups fileSys.delete(outDir, true); jobConf.setOutputCommitter(CommitterWithFailTaskCleanup.class); rJob = launchJob(jobConf, inDir, outDir, input); rJob.waitForCompletion(); validateJob(rJob, jt, true); fileSys.delete(outDir, true); jobConf.setOutputCommitter(CommitterWithFailTaskCleanup2.class); rJob = launchJob(jobConf, inDir, outDir, input); rJob.waitForCompletion(); validateJob(rJob, jt, true); // launch job with task-cleanup switched off fileSys.delete(outDir, true); jobConf.setOutputCommitter(CommitterWithFailTaskCleanup.class); jobConf.setBoolean(MRJobConfig.TASK_CLEANUP_NEEDED, false); rJob = launchJob(jobConf, inDir, outDir, input); rJob.waitForCompletion(); validateJob(rJob, jt, false); } finally { if (dfs != null) { dfs.shutdown(); } if (mr != null) { mr.shutdown(); } }
5482 -1009652244apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None create input split for remaining items if necessary this includes the case where no splits were created by the loop SATD_ADDED filesPerTask(Configuration) protected static long filesPerTask(Configuration conf) return conf.getLong(BLOCKFIX_FILES_PER_TASK, DEFAULT_BLOCKFIX_FILES_PER_TASK);
5479 -1009652247apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None TODO: this is conservative. SATD_ADDED map(LongWritable, IntWritable, Context) protected void map(LongWritable offset, IntWritable length, final Context context) throws IOException, InterruptedException LOG.info("offset=" + offset + ", length=" + length); // compute digits final byte[] bytes = new byte[length.get() >> 1]; long d = offset.get(); for (int i = 0; i < bytes.length; d += 4) { final long digits = hexDigits(d); bytes[i++] = (byte) (digits >> 8); bytes[i++] = (byte) digits; } // output map results context.write(offset, new BytesWritable(bytes));
5478 -1009652248apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None create Configs.SYSTEM_DIR's parent with restrictive permissions. So long as the JT has access to the system dir itself it should be able to start. SATD_ADDED testGarbledMapredSystemDir() public void testGarbledMapredSystemDir() throws Exception Configuration conf = new Configuration(); final MiniDFSCluster dfs = new MiniDFSCluster(conf, 1, true, null); MiniMRCluster mr = null; try { // start dfs conf.set("dfs.permissions.supergroup", "supergroup"); FileSystem fs = DFS_UGI.doAs(new PrivilegedExceptionAction() { public FileSystem run() throws IOException { return dfs.getFileSystem(); } }); // create Configs.SYSTEM_DIR's parent with restrictive permissions. // So long as the JT has access to the system dir itself it should // be able to start. Path mapredSysDir = new Path(conf.get(JTConfig.JT_SYSTEM_DIR)); Path parentDir = mapredSysDir.getParent(); fs.mkdirs(parentDir); fs.setPermission(parentDir, new FsPermission(SYSTEM_DIR_PARENT_PERMISSION)); fs.mkdirs(mapredSysDir); fs.setPermission(mapredSysDir, new FsPermission(SYSTEM_DIR_PERMISSION)); fs.setOwner(mapredSysDir, "mr", "mrgroup"); // start mr (i.e jobtracker) Configuration mrConf = new Configuration(conf); mr = new MiniMRCluster(0, 0, 0, dfs.getFileSystem().getUri().toString(), 1, null, null, MR_UGI, new JobConf(mrConf)); JobTracker jobtracker = mr.getJobTrackerRunner().getJobTracker(); // add garbage to Configs.SYSTEM_DIR Path garbage = new Path(jobtracker.getSystemDir(), "garbage"); fs.mkdirs(garbage); fs.setPermission(garbage, new FsPermission(SYSTEM_DIR_PERMISSION)); fs.setOwner(garbage, "test", "test-group"); // stop the jobtracker mr.stopJobTracker(); mr.getJobTrackerConf().setBoolean(JTConfig.JT_RESTART_ENABLED, false); // start jobtracker but dont wait for it to be up mr.startJobTracker(false); // check 5 times .. each time wait for 2 secs to check if the jobtracker // has crashed or not. for (int i = 0; i < 5; ++i) { LOG.info("Check #" + i); if (!mr.getJobTrackerRunner().isActive()) { return; } UtilsForTests.waitFor(2000); } assertFalse("JobTracker did not bail out (waited for 10 secs)", mr.getJobTrackerRunner().isActive()); } finally { if (dfs != null) { dfs.shutdown(); } if (mr != null) { mr.shutdown(); } }
5477 -1009652249apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Shouldn't ever hit this case. SATD_ADDED longToDate(long, int) private Date longToDate(long val, int sqlDataType) switch(sqlDataType) { case Types.DATE: return new java.sql.Date(val); case Types.TIME: return new java.sql.Time(val); case Types.TIMESTAMP: return new java.sql.Timestamp(val); default: // Shouldn't ever hit this case. return null; }
5476 -1009652250apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None The range of acceptable dates is NULL to NULL. Just create a single split. SATD_ADDED split(Configuration, ResultSet, String) public List split(Configuration conf, ResultSet results, String colName) throws SQLException long minVal; long maxVal; int sqlDataType = results.getMetaData().getColumnType(1); minVal = resultSetColToLong(results, 1, sqlDataType); maxVal = resultSetColToLong(results, 2, sqlDataType); String lowClausePrefix = colName + " >= "; String highClausePrefix = colName + " < "; int numSplits = conf.getInt(MRJobConfig.NUM_MAPS, 1); if (numSplits < 1) { numSplits = 1; } if (minVal == Long.MIN_VALUE && maxVal == Long.MIN_VALUE) { // The range of acceptable dates is NULL to NULL. Just create a single split. List splits = new ArrayList(); splits.add(new DataDrivenDBInputFormat.DataDrivenDBInputSplit(colName + " IS NULL", colName + " IS NULL")); return splits; } // Gather the split point integers List splitPoints = split(numSplits, minVal, maxVal); List splits = new ArrayList(); // Turn the split points into a set of intervals. long start = splitPoints.get(0); Date startDate = longToDate(start, sqlDataType); if (sqlDataType == Types.TIMESTAMP) { // The lower bound's nanos value needs to match the actual lower-bound nanos. try { ((java.sql.Timestamp) startDate).setNanos(results.getTimestamp(1).getNanos()); } catch (NullPointerException npe) { // If the lower bound was NULL, we'll get an NPE; just ignore it and don't set nanos. } } for (int i = 1; i < splitPoints.size(); i++) { long end = splitPoints.get(i); Date endDate = longToDate(end, sqlDataType); if (i == splitPoints.size() - 1) { if (sqlDataType == Types.TIMESTAMP) { // The upper bound's nanos value needs to match the actual upper-bound nanos. try { ((java.sql.Timestamp) endDate).setNanos(results.getTimestamp(2).getNanos()); } catch (NullPointerException npe) { // If the upper bound was NULL, we'll get an NPE; just ignore it and don't set nanos. } } // This is the last one; use a closed interval. splits.add(new DataDrivenDBInputFormat.DataDrivenDBInputSplit(lowClausePrefix + dateToString(startDate), colName + " <= " + dateToString(endDate))); } else { // Normal open-interval case. splits.add(new DataDrivenDBInputFormat.DataDrivenDBInputSplit(lowClausePrefix + dateToString(startDate), highClausePrefix + dateToString(endDate))); } start = end; startDate = endDate; } if (minVal == Long.MIN_VALUE || maxVal == Long.MIN_VALUE) { // Add an extra split to handle the null case that we saw. splits.add(new DataDrivenDBInputFormat.DataDrivenDBInputSplit(colName + " IS NULL", colName + " IS NULL")); } return splits;
5475 -1009652251apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None TODO add listeners, use for job dependencies SATD_ADDED getGridmixInputDataPath(Path) static Path getGridmixInputDataPath(Path ioPath) return new Path(ioPath, "input");
5474 -1009652252apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None We need this odd-looking code because if a seed exists we need to ensure that only one interpolator is generated per LoggedDiscreteCDF, but if no seed exists then the potentially lengthy process of making an interpolator can happen outside the lock. makeUpRuntimeCore only locks around the two hash map accesses. SATD_ADDED convertState(Values) private static State convertState(Values status) if (status == Values.SUCCESS) { return State.SUCCEEDED; } else if (status == Values.FAILED) { return State.FAILED; } else if (status == Values.KILLED) { return State.KILLED; } else { throw new IllegalArgumentException("unknown status " + status); }
5473 -1009652253apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None TODO make up state SATD_ADDED convertState(Values) private static State convertState(Values status) if (status == Values.SUCCESS) { return State.SUCCEEDED; } else if (status == Values.FAILED) { return State.FAILED; } else if (status == Values.KILLED) { return State.KILLED; } else { throw new IllegalArgumentException("unknown status " + status); }
5472 -1009652254apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None TODO should we handle killed attempts later? SATD_ADDED convertState(Values) private static State convertState(Values status) if (status == Values.SUCCESS) { return State.SUCCEEDED; } else if (status == Values.FAILED) { return State.FAILED; } else if (status == Values.KILLED) { return State.KILLED; } else { throw new IllegalArgumentException("unknown status " + status); }
5471 -1009652255apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None TODO insert parameters SATD_ADDED convertState(Values) private static State convertState(Values status) if (status == Values.SUCCESS) { return State.SUCCEEDED; } else if (status == Values.FAILED) { return State.FAILED; } else if (status == Values.KILLED) { return State.KILLED; } else { throw new IllegalArgumentException("unknown status " + status); }
5470 -1009652256apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None TODO: do not care about \"other\" tasks, \"setup\" or \"clean\" SATD_ADDED convertState(Values) private static State convertState(Values status) if (status == Values.SUCCESS) { return State.SUCCEEDED; } else if (status == Values.FAILED) { return State.FAILED; } else if (status == Values.KILLED) { return State.KILLED; } else { throw new IllegalArgumentException("unknown status " + status); }
5469 -1009652257apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None TODO set size of a split to 0 now. SATD_ADDED convertState(Values) private static State convertState(Values status) if (status == Values.SUCCESS) { return State.SUCCEEDED; } else if (status == Values.FAILED) { return State.FAILED; } else if (status == Values.KILLED) { return State.KILLED; } else { throw new IllegalArgumentException("unknown status " + status); }
5468 -1009652258apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None TODO Eliminate parameters that are already copied from the job's configuration file. SATD_ADDED convertState(Values) private static State convertState(Values status) if (status == Values.SUCCESS) { return State.SUCCEEDED; } else if (status == Values.FAILED) { return State.FAILED; } else if (status == Values.KILLED) { return State.KILLED; } else { throw new IllegalArgumentException("unknown status " + status); }
5466 -1009652260apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None TODO Add a layer to enable SQL \"sharding\" and support locality SATD_ADDED readFields(DataInput) public void readFields(DataInput in) throws IOException
5465 -1009652261apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None test hashing distribution policy SATD_ADDED testDistributionPolicy() public void testDistributionPolicy() throws IOException IndexUpdateConfiguration iconf = new IndexUpdateConfiguration(conf); // test hashing distribution policy iconf.setDistributionPolicyClass(HashingDistributionPolicy.class); onetest(); if (fs.exists(indexPath)) { fs.delete(indexPath, true); } // test round-robin distribution policy iconf.setDistributionPolicyClass(RoundRobinDistributionPolicy.class); onetest();
5464 -1009652262apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None HACK ALERT!! It's possible for a Job end line to end a job for which we have a config file image [ a ParsedConfigFile ] in jobconf. processParsedLine handles this. SATD_ADDED run() Pair line = readBalancedLine(); while (line != null) { if (debug && (lineNumber < 1000000L && lineNumber % 1000L == 0 || lineNumber % 1000000L == 0)) { LOG.debug("" + lineNumber + " " + line.second()); } if (line.first() == null) { try { // HACK ALERT!! It's possible for a Job end line to end a // job for which we have a config file // image [ a ParsedConfigFile ] in jobconf. // // processParsedLine handles this. processParsedLine(new ParsedLine(line.second(), version)); } catch (StringIndexOutOfBoundsException e) { LOG.warn("anomalous line #" + lineNumber + ":" + line, e); } } else { jobconf = new ParsedConfigFile(line.first(), line.second()); if (jobconf.valid == false) { jobconf = null; } maybeMateJobAndConf(); } line = readBalancedLine(); } finalizeJob(); if (collecting) { String[] typeNames = LogRecordType.lineTypes(); for (int i = 0; i < typeNames.length; ++i) { statisticalOutput.print(typeNames[i]); statisticalOutput.print('\n'); } } else { if (delays) { printDistributionSet("Job start delay spectrum:", delayTimeDists); } if (runtimes) { printDistributionSet("Job run time spectrum:", runTimeDists); } if (spreading) { String ratioDescription = "(" + spreadMax + "/1000 %ile) to (" + spreadMin + "/1000 %ile) scaled by 1000000"; printDistributionSet("Map task success times " + ratioDescription + ":", mapTimeSpreadDists); printDistributionSet("Shuffle success times " + ratioDescription + ":", shuffleTimeSpreadDists); printDistributionSet("Sort success times " + ratioDescription + ":", sortTimeSpreadDists); printDistributionSet("Reduce success times " + ratioDescription + ":", reduceTimeSpreadDists); } if (collectTaskTimes) { printDistributionSet("Global map task success times:", mapTimeDists); printDistributionSet("Global shuffle task success times:", shuffleTimeDists); printDistributionSet("Global sort task success times:", sortTimeDists); printDistributionSet("Global reduce task success times:", reduceTimeDists); } } if (topologyGen != null) { LoggedNetworkTopology topo = new LoggedNetworkTopology(allHosts, "", 0); topologyGen.writeObject(topo); topologyGen.close(); } if (jobTraceGen != null) { jobTraceGen.close(); } if (input != null) { input.close(); input = null; } if (inputCodec != null) { CodecPool.returnDecompressor(inputDecompressor); inputDecompressor = null; inputCodec = null; } return 0; int run() throws IOException
5463 -1009652263apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None there are some literals here that probably should be options SATD_ADDED newDistributionBlock() private Histogram[][] newDistributionBlock() return newDistributionBlock(null);
5461 -1009652265apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None we first start a cluster and fill the cluster up to a certain size. then redistribute blocks according the required distribution. Afterwards a balancer is running to balance the cluster. SATD_ADDED testUnevenDistribution(Configuration, long[], long[], String[]) private void testUnevenDistribution(Configuration conf, long[] distribution, long[] capacities, String[] racks) throws Exception int numDatanodes = distribution.length; if (capacities.length != numDatanodes || racks.length != numDatanodes) { throw new IllegalArgumentException("Array length is not the same"); } // calculate total space that need to be filled final long totalUsedSpace = sum(distribution); // fill the cluster ExtendedBlock[] blocks = generateBlocks(conf, totalUsedSpace, (short) numDatanodes); // redistribute blocks Block[][] blocksDN = distributeBlocks(blocks, (short) (numDatanodes - 1), distribution); // restart the cluster: do NOT format the cluster conf.set(DFSConfigKeys.DFS_NAMENODE_SAFEMODE_THRESHOLD_PCT_KEY, "0.0f"); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDatanodes).format(false).racks(racks).simulatedCapacities(capacities).build(); cluster.waitActive(); client = DFSClient.createNamenode(conf); for (int i = 0; i < blocksDN.length; i++) cluster.injectBlocks(i, Arrays.asList(blocksDN[i])); final long totalCapacity = sum(capacities); runBalancer(conf, totalUsedSpace, totalCapacity); cluster.shutdown();
5460 -1009652266apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None consider job-submitter for private distributed cache file SATD_ADDED mapDistCacheFilePath(String, String, boolean, String) private String mapDistCacheFilePath(String file, String timeStamp, boolean isPublic, String user) String id = file + timeStamp; if (!isPublic) { // consider job-submitter for private distributed cache file id = id.concat(user); } return new Path(distCachePath, MD5Hash.digest(id).toString()).toUri().getPath();
5457 -1009652269apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None There is high probability that the rack id generated here will not conflict with those of other data node cluster. Not perfect but mostly unique rack ids are good enough SATD_ADDED getUniqueRackPrefix() private static String getUniqueRackPrefix() String ip = "unknownIP"; try { ip = DNS.getDefaultIP("default"); } catch (UnknownHostException ignored) { System.out.println("Could not find ip address of \"default\" inteface."); } int rand = 0; try { rand = SecureRandom.getInstance("SHA1PRNG").nextInt(Integer.MAX_VALUE); } catch (NoSuchAlgorithmException e) { rand = (new Random()).nextInt(Integer.MAX_VALUE); } return "/Rack-" + rand + "-" + ip + "-" + System.currentTimeMillis();
5456 -1009652270apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None TODO close DFS connections? SATD_ADDED dispose() public void dispose() // TODO close DFS connections?
5454 -1009652272apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Refresh the service level authorization policy once again, this time it should fail! SATD_ADDED testRefresh() public void testRefresh() throws Exception MiniDFSCluster dfs = null; try { final int slaves = 4; // Turn on service-level authorization final Configuration conf = new Configuration(); conf.setClass(PolicyProvider.POLICY_PROVIDER_CONFIG, HDFSPolicyProvider.class, PolicyProvider.class); conf.setBoolean(ServiceAuthorizationManager.SERVICE_AUTHORIZATION_CONFIG, true); // Start the mini dfs cluster dfs = new MiniDFSCluster(conf, slaves, true, null); // Refresh the service level authorization policy refreshPolicy(conf); // Simulate an 'edit' of hadoop-policy.xml String confDir = System.getProperty("test.build.extraconf", "build/test/extraconf"); String HADOOP_POLICY_FILE = System.getProperty("hadoop.policy.file"); File policyFile = new File(confDir, HADOOP_POLICY_FILE); String policyFileCopy = HADOOP_POLICY_FILE + ".orig"; // first save original FileUtil.copy(// first save original policyFile, // first save original FileSystem.getLocal(conf), new Path(confDir, policyFileCopy), false, conf); rewriteHadoopPolicyFile(new File(confDir, HADOOP_POLICY_FILE)); // Refresh the service level authorization policy refreshPolicy(conf); // Refresh the service level authorization policy once again, // this time it should fail! try { // Note: hadoop-policy.xml for tests has // security.refresh.policy.protocol.acl = ${user.name} UserGroupInformation unknownUser = UserGroupInformation.createRemoteUser("unknown"); unknownUser.doAs(new PrivilegedExceptionAction() { public Void run() throws IOException { refreshPolicy(conf); return null; } }); fail("Refresh of NameNode's policy file cannot be successful!"); } catch (Exception re) { System.out.println("Good, refresh worked... refresh failed with: " + StringUtils.stringifyException(re)); } finally { // Reset to original hadoop-policy.xml FileUtil.fullyDelete(new File(confDir, HADOOP_POLICY_FILE)); FileUtil.replaceFile(new File(confDir, policyFileCopy), new File(confDir, HADOOP_POLICY_FILE)); } } finally { if (dfs != null) { dfs.shutdown(); } }
5453 -1009652273apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Refresh the service level authorization policy SATD_ADDED testRefresh() public void testRefresh() throws Exception MiniDFSCluster dfs = null; try { final int slaves = 4; // Turn on service-level authorization final Configuration conf = new Configuration(); conf.setClass(PolicyProvider.POLICY_PROVIDER_CONFIG, HDFSPolicyProvider.class, PolicyProvider.class); conf.setBoolean(ServiceAuthorizationManager.SERVICE_AUTHORIZATION_CONFIG, true); // Start the mini dfs cluster dfs = new MiniDFSCluster(conf, slaves, true, null); // Refresh the service level authorization policy refreshPolicy(conf); // Simulate an 'edit' of hadoop-policy.xml String confDir = System.getProperty("test.build.extraconf", "build/test/extraconf"); String HADOOP_POLICY_FILE = System.getProperty("hadoop.policy.file"); File policyFile = new File(confDir, HADOOP_POLICY_FILE); String policyFileCopy = HADOOP_POLICY_FILE + ".orig"; // first save original FileUtil.copy(// first save original policyFile, // first save original FileSystem.getLocal(conf), new Path(confDir, policyFileCopy), false, conf); rewriteHadoopPolicyFile(new File(confDir, HADOOP_POLICY_FILE)); // Refresh the service level authorization policy refreshPolicy(conf); // Refresh the service level authorization policy once again, // this time it should fail! try { // Note: hadoop-policy.xml for tests has // security.refresh.policy.protocol.acl = ${user.name} UserGroupInformation unknownUser = UserGroupInformation.createRemoteUser("unknown"); unknownUser.doAs(new PrivilegedExceptionAction() { public Void run() throws IOException { refreshPolicy(conf); return null; } }); fail("Refresh of NameNode's policy file cannot be successful!"); } catch (Exception re) { System.out.println("Good, refresh worked... refresh failed with: " + StringUtils.stringifyException(re)); } finally { // Reset to original hadoop-policy.xml FileUtil.fullyDelete(new File(confDir, HADOOP_POLICY_FILE)); FileUtil.replaceFile(new File(confDir, policyFileCopy), new File(confDir, HADOOP_POLICY_FILE)); } } finally { if (dfs != null) { dfs.shutdown(); } }
5452 -1009652274apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Stop and start the tracker. The tracker should come up nicely SATD_ADDED testHistoryInitWithCorruptFiles() public void testHistoryInitWithCorruptFiles() throws IOException MiniMRCluster mr = null; try { JobConf conf = new JobConf(); Path historyDir = new Path(System.getProperty("test.build.data", "."), "history"); conf.set(JTConfig.JT_JOBHISTORY_LOCATION, historyDir.toString()); conf.setUser("user"); FileSystem localFs = FileSystem.getLocal(conf); // there may be some stale files, clean them if (localFs.exists(historyDir)) { boolean deleted = localFs.delete(historyDir, true); LOG.info(historyDir + " deleted " + deleted); } // Start the cluster, create a history file mr = new MiniMRCluster(0, "file:///", 3, null, null, conf); JobTracker jt = mr.getJobTrackerRunner().getJobTracker(); JobHistory jh = jt.getJobHistory(); final JobID jobId = JobID.forName("job_200809171136_0001"); jh.setupEventWriter(jobId, conf); Map jobACLs = new HashMap(); JobSubmittedEvent jse = new JobSubmittedEvent(jobId, "job", "user", 12345, "path", jobACLs, "default"); jh.logEvent(jse, jobId); jh.closeWriter(jobId); // Corrupt the history file. User RawLocalFileSystem so that we // do keep the original CRC file intact. String historyFileName = jobId.toString() + "_" + "user"; Path historyFilePath = new Path(historyDir.toString(), historyFileName); RawLocalFileSystem fs = (RawLocalFileSystem) FileSystem.getLocal(conf).getRaw(); FSDataOutputStream out = fs.create(historyFilePath, true); byte[] corruptData = new byte[32]; new Random().nextBytes(corruptData); out.write(corruptData, 0, 32); out.close(); // Stop and start the tracker. The tracker should come up nicely mr.stopJobTracker(); mr.startJobTracker(); jt = mr.getJobTrackerRunner().getJobTracker(); assertNotNull("JobTracker did not come up", jt); jh = jt.getJobHistory(); assertNotNull("JobHistory did not get initialized correctly", jh); // Only the done folder should remain in the history directory assertEquals("Files in logDir did not move to DONE folder", 1, historyDir.getFileSystem(conf).listStatus(historyDir).length); } finally { if (mr != null) { cleanupLocalFiles(mr); mr.shutdown(); } }
5451 -1009652275apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None TODO this is a hack :( jobtracker-hostname_jobtracker-identifier_ SATD_ADDED getPathForConf(Path, Path) private static Path getPathForConf(Path path, Path dir) String[] parts = path.getName().split("_"); // TODO this is a hack :( // jobtracker-hostname_jobtracker-identifier_ String id = parts[0] + "_" + parts[1] + "_" + parts[2]; return new Path(dir, id + "_conf.xml");
5450 -1009652276apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Check if the job has been properly cleaned up. SATD_ADDED testFailedJobInitalizations() public void testFailedJobInitalizations() throws Exception String[] qs = { "default" }; taskTrackerManager.addQueues(qs); ArrayList queues = new ArrayList(); queues.add(new FakeQueueInfo("default", 100.0f, true, 100)); taskTrackerManager.setFakeQueues(queues); scheduler.start(); JobQueuesManager mgr = scheduler.jobQueuesManager; // Submit a job whose initialization would fail always. FakeJobInProgress job = new FakeFailingJobInProgress(new JobID("test", ++jobCounter), new JobConf(), taskTrackerManager, "u1", UtilsForTests.getJobTracker()); job.getStatus().setRunState(JobStatus.PREP); taskTrackerManager.submitJob(job); // check if job is present in waiting list. assertEquals("Waiting job list does not contain submitted job", 1, mgr.getJobQueue("default").getWaitingJobCount()); assertTrue("Waiting job does not contain submitted job", mgr.getJobQueue("default").getWaitingJobs().contains(job)); // initialization should fail now. controlledInitializationPoller.selectJobsToInitialize(); // Check if the job has been properly cleaned up. assertEquals("Waiting job list contains submitted job", 0, mgr.getJobQueue("default").getWaitingJobCount()); assertFalse("Waiting job contains submitted job", mgr.getJobQueue("default").getWaitingJobs().contains(job)); assertFalse("Waiting job contains submitted job", mgr.getJobQueue("default").getRunningJobs().contains(job));
5449 -1009652277apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Total 1 map slot should be accounted for. SATD_ADDED testClusterBlockingForLackOfMemory() public void testClusterBlockingForLackOfMemory() throws IOException LOG.debug("Starting the scheduler."); taskTrackerManager = new FakeTaskTrackerManager(3, 2, 2); ArrayList queues = new ArrayList(); queues.add(new FakeQueueInfo("default", 100.0f, true, 25)); taskTrackerManager.addQueues(new String[] { "default" }); scheduler.setTaskTrackerManager(taskTrackerManager); // enabled memory-based scheduling // Normal jobs 1GB maps/reduces. 2GB limit on maps/reduces scheduler.getConf().setLong(JTConfig.JT_MAX_MAPMEMORY_MB, 2 * 1024); scheduler.getConf().setLong(MRConfig.MAPMEMORY_MB, 1 * 1024); scheduler.getConf().setLong(JTConfig.JT_MAX_REDUCEMEMORY_MB, 2 * 1024); scheduler.getConf().setLong(MRConfig.REDUCEMEMORY_MB, 1 * 1024); taskTrackerManager.setFakeQueues(queues); scheduler.start(); LOG.debug("Submit one normal memory(1GB maps/reduces) job of " + "2 map, 2 reduce tasks."); JobConf jConf = new JobConf(conf); jConf.setMemoryForMapTask(1 * 1024); jConf.setMemoryForReduceTask(1 * 1024); jConf.setNumMapTasks(2); jConf.setNumReduceTasks(2); jConf.setQueueName("default"); jConf.setUser("u1"); FakeJobInProgress job1 = taskTrackerManager.submitJobAndInit(JobStatus.PREP, jConf); // Fill a tt with this job's tasks. Map expectedStrings = new HashMap(); expectedStrings.put(CapacityTestUtils.MAP, "attempt_test_0001_m_000001_0 on tt1"); expectedStrings.put(CapacityTestUtils.REDUCE, "attempt_test_0001_r_000001_0 on tt1"); checkMultipleTaskAssignment(taskTrackerManager, scheduler, "tt1", expectedStrings); // Total 1 map slot should be accounted for. checkOccupiedSlots("default", TaskType.MAP, 1, 1, 16.7f); checkOccupiedSlots("default", TaskType.REDUCE, 1, 1, 16.7f); assertEquals(JobQueue.getJobQueueSchedInfo(1, 1, 0, 1, 1, 0), job1.getSchedulingInfo().toString()); checkMemReservedForTasksOnTT("tt1", 1 * 1024L, 1 * 1024L); expectedStrings.clear(); expectedStrings.put(CapacityTestUtils.MAP, "attempt_test_0001_m_000002_0 on tt2"); expectedStrings.put(CapacityTestUtils.REDUCE, "attempt_test_0001_r_000002_0 on tt2"); // fill another TT with the rest of the tasks of the job checkMultipleTaskAssignment(taskTrackerManager, scheduler, "tt2", expectedStrings); LOG.debug("Submit one high memory(2GB maps/reduces) job of " + "2 map, 2 reduce tasks."); jConf = new JobConf(conf); jConf.setMemoryForMapTask(2 * 1024); jConf.setMemoryForReduceTask(2 * 1024); jConf.setNumMapTasks(2); jConf.setNumReduceTasks(2); jConf.setQueueName("default"); jConf.setUser("u1"); FakeJobInProgress job2 = taskTrackerManager.submitJobAndInit(JobStatus.PREP, jConf); // Have another TT run one task of each type of the high RAM // job. This will fill up the TT. expectedStrings.clear(); expectedStrings.put(CapacityTestUtils.MAP, "attempt_test_0002_m_000001_0 on tt3"); expectedStrings.put(CapacityTestUtils.REDUCE, "attempt_test_0002_r_000001_0 on tt3"); checkMultipleTaskAssignment(taskTrackerManager, scheduler, "tt3", expectedStrings); checkOccupiedSlots("default", TaskType.MAP, 1, 4, 66.7f); checkOccupiedSlots("default", TaskType.REDUCE, 1, 4, 66.7f); assertEquals(JobQueue.getJobQueueSchedInfo(1, 2, 0, 1, 2, 0), job2.getSchedulingInfo().toString()); checkMemReservedForTasksOnTT("tt3", 2 * 1024L, 2 * 1024L); LOG.debug("Submit one normal memory(1GB maps/reduces) job of " + "1 map, 1 reduce tasks."); jConf = new JobConf(conf); jConf.setMemoryForMapTask(1 * 1024); jConf.setMemoryForReduceTask(1 * 1024); jConf.setNumMapTasks(1); jConf.setNumReduceTasks(1); jConf.setQueueName("default"); jConf.setUser("u1"); FakeJobInProgress job3 = taskTrackerManager.submitJobAndInit(JobStatus.PREP, jConf); // Send a TT with insufficient space for task assignment, // This will cause a reservation for the high RAM job. assertNull(scheduler.assignTasks(tracker("tt1"))); // reserved tasktrackers contribute to occupied slots for maps and reduces checkOccupiedSlots("default", TaskType.MAP, 1, 6, 100.0f); checkOccupiedSlots("default", TaskType.REDUCE, 1, 6, 100.0f); checkMemReservedForTasksOnTT("tt1", 1 * 1024L, 1 * 1024L); LOG.info(job2.getSchedulingInfo()); assertEquals(JobQueue.getJobQueueSchedInfo(1, 2, 2, 1, 2, 2), job2.getSchedulingInfo().toString()); assertEquals(JobQueue.getJobQueueSchedInfo(0, 0, 0, 0, 0, 0), job3.getSchedulingInfo().toString()); // Reservations are already done for job2. So job3 should go ahead. expectedStrings.clear(); expectedStrings.put(CapacityTestUtils.MAP, "attempt_test_0003_m_000001_0 on tt2"); expectedStrings.put(CapacityTestUtils.REDUCE, "attempt_test_0003_r_000001_0 on tt2"); checkMultipleTaskAssignment(taskTrackerManager, scheduler, "tt2", expectedStrings);
5448 -1009652278apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Total 2 map slots should be accounted for. SATD_ADDED testHighMemoryBlockingAcrossTaskTypes() public void testHighMemoryBlockingAcrossTaskTypes() throws IOException // 2 map and 1 reduce slots taskTrackerManager = new FakeTaskTrackerManager(1, 2, 2); taskTrackerManager.addQueues(new String[] { "default" }); ArrayList queues = new ArrayList(); queues.add(new FakeQueueInfo("default", 100.0f, true, 25)); scheduler.setTaskTrackerManager(taskTrackerManager); // enabled memory-based scheduling // Normal job in the cluster would be 1GB maps/reduces scheduler.getConf().setLong(JTConfig.JT_MAX_MAPMEMORY_MB, 2 * 1024); scheduler.getConf().setLong(MRConfig.MAPMEMORY_MB, 1 * 1024); scheduler.getConf().setLong(JTConfig.JT_MAX_REDUCEMEMORY_MB, 1 * 1024); scheduler.getConf().setLong(MRConfig.REDUCEMEMORY_MB, 1 * 1024); taskTrackerManager.setFakeQueues(queues); scheduler.start(); // The situation : Two jobs in the queue. First job with only maps and no // reduces and is a high memory job. Second job is a normal job with both // maps and reduces. // First job cannot run for want of memory for maps. In this case, second // job's reduces should run. LOG.debug("Submit one high memory(2GB maps, 0MB reduces) job of " + "2 map tasks"); JobConf jConf = new JobConf(conf); jConf.setMemoryForMapTask(2 * 1024); jConf.setMemoryForReduceTask(0); jConf.setNumMapTasks(2); jConf.setNumReduceTasks(0); jConf.setQueueName("default"); jConf.setUser("u1"); FakeJobInProgress job1 = taskTrackerManager.submitJobAndInit(JobStatus.PREP, jConf); LOG.debug("Submit another regular memory(1GB vmem maps/reduces) job of " + "2 map/red tasks"); jConf = new JobConf(conf); jConf.setMemoryForMapTask(1 * 1024); jConf.setMemoryForReduceTask(1 * 1024); jConf.setNumMapTasks(2); jConf.setNumReduceTasks(2); jConf.setQueueName("default"); jConf.setUser("u1"); FakeJobInProgress job2 = taskTrackerManager.submitJobAndInit(JobStatus.PREP, jConf); // first, a map from j1 and a reduce from other job j2 Map strs = new HashMap(); strs.put(MAP, "attempt_test_0001_m_000001_0 on tt1"); strs.put(REDUCE, "attempt_test_0002_r_000001_0 on tt1"); checkMultipleTaskAssignment(taskTrackerManager, scheduler, "tt1", strs); // Total 2 map slots should be accounted for. checkOccupiedSlots("default", TaskType.MAP, 1, 2, 100.0f); checkOccupiedSlots("default", TaskType.REDUCE, 1, 1, 50.0f); checkMemReservedForTasksOnTT("tt1", 2 * 1024L, 1 * 1024L); // TT has 2 slots for reduces hence this call should get a reduce task // from other job checkAssignment(taskTrackerManager, scheduler, "tt1", "attempt_test_0002_r_000002_0 on tt1"); checkOccupiedSlots("default", TaskType.MAP, 1, 2, 100.0f); checkOccupiedSlots("default", TaskType.REDUCE, 1, 2, 100.0f); checkMemReservedForTasksOnTT("tt1", 2 * 1024L, 2 * 1024L); // now as all the slots are occupied hence no more tasks would be // assigned. assertNull(scheduler.assignTasks(tracker("tt1")));
5447 -1009652279apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None u1 has 5 map slots and 5 reduce slots. u2 has 4 map slots and 4 reduce slots. Because of high memory tasks, giving u2 another task would overflow limits. So, no more tasks should be given to anyone. SATD_ADDED testUserLimitsForHighMemoryJobs() public void testUserLimitsForHighMemoryJobs() throws IOException taskTrackerManager = new FakeTaskTrackerManager(1, 10, 10); scheduler.setTaskTrackerManager(taskTrackerManager); String[] qs = { "default" }; taskTrackerManager.addQueues(qs); ArrayList queues = new ArrayList(); queues.add(new FakeQueueInfo("default", 100.0f, true, 50)); // enabled memory-based scheduling // Normal job in the cluster would be 1GB maps/reduces scheduler.getConf().setLong(JTConfig.JT_MAX_MAPMEMORY_MB, 2 * 1024); scheduler.getConf().setLong(MRConfig.MAPMEMORY_MB, 1 * 1024); scheduler.getConf().setLong(JTConfig.JT_MAX_REDUCEMEMORY_MB, 2 * 1024); scheduler.getConf().setLong(MRConfig.REDUCEMEMORY_MB, 1 * 1024); taskTrackerManager.setFakeQueues(queues); scheduler.start(); // Submit one normal job to the other queue. JobConf jConf = new JobConf(conf); jConf.setMemoryForMapTask(1 * 1024); jConf.setMemoryForReduceTask(1 * 1024); jConf.setNumMapTasks(6); jConf.setNumReduceTasks(6); jConf.setUser("u1"); jConf.setQueueName("default"); FakeJobInProgress job1 = taskTrackerManager.submitJobAndInit(JobStatus.PREP, jConf); LOG.debug("Submit one high memory(2GB maps, 2GB reduces) job of " + "6 map and 6 reduce tasks"); jConf = new JobConf(conf); jConf.setMemoryForMapTask(2 * 1024); jConf.setMemoryForReduceTask(2 * 1024); jConf.setNumMapTasks(6); jConf.setNumReduceTasks(6); jConf.setQueueName("default"); jConf.setUser("u2"); FakeJobInProgress job2 = taskTrackerManager.submitJobAndInit(JobStatus.PREP, jConf); // Verify that normal job takes 5 task assignments to hit user limits Map expectedStrings = new HashMap(); for (int i = 0; i < 5; i++) { expectedStrings.clear(); expectedStrings.put(CapacityTestUtils.MAP, "attempt_test_0001_m_00000" + (i + 1) + "_0 on tt1"); expectedStrings.put(CapacityTestUtils.REDUCE, "attempt_test_0001_r_00000" + (i + 1) + "_0 on tt1"); checkMultipleTaskAssignment(taskTrackerManager, scheduler, "tt1", expectedStrings); } // u1 has 5 map slots and 5 reduce slots. u2 has none. So u1's user limits // are hit. So u2 should get slots for (int i = 0; i < 2; i++) { expectedStrings.clear(); expectedStrings.put(CapacityTestUtils.MAP, "attempt_test_0002_m_00000" + (i + 1) + "_0 on tt1"); expectedStrings.put(CapacityTestUtils.REDUCE, "attempt_test_0002_r_00000" + (i + 1) + "_0 on tt1"); checkMultipleTaskAssignment(taskTrackerManager, scheduler, "tt1", expectedStrings); } // u1 has 5 map slots and 5 reduce slots. u2 has 4 map slots and 4 reduce // slots. Because of high memory tasks, giving u2 another task would // overflow limits. So, no more tasks should be given to anyone. assertNull(scheduler.assignTasks(tracker("tt1")));
5446 -1009652280apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None by doing the above clock adjustments, we bring the progress rate of taskID 3 lower than 4. For taskID 3, the rate is 85/317000 and for taskID 4, the rate is 20/65000. But when we ask for a spec task now, we should get back taskID 4 (since that is expected to complete later than taskID 3). SATD_ADDED testTaskLATEScheduling() public void testTaskLATEScheduling() throws IOException TaskAttemptID[] taskAttemptID = new TaskAttemptID[20]; JobConf conf = new JobConf(); conf.setSpeculativeExecution(true); conf.setNumMapTasks(5); conf.setNumReduceTasks(0); conf.setFloat(JobContext.SPECULATIVE_SLOWTASK_THRESHOLD, 0.5f); FakeJobInProgress job = new FakeJobInProgress(conf, jobTracker); job.initTasks(); taskAttemptID[0] = job.findMapTask(trackers[0]); taskAttemptID[1] = job.findMapTask(trackers[1]); taskAttemptID[2] = job.findMapTask(trackers[2]); taskAttemptID[3] = job.findMapTask(trackers[3]); clock.advance(2000); job.finishTask(taskAttemptID[0]); job.finishTask(taskAttemptID[1]); job.finishTask(taskAttemptID[2]); clock.advance(250000); taskAttemptID[4] = job.findMapTask(trackers[3]); clock.advanceBySpeculativeLag(); // by doing the above clock adjustments, we bring the progress rate of // taskID 3 lower than 4. For taskID 3, the rate is 85/317000 // and for taskID 4, the rate is 20/65000. But when we ask for a spec task // now, we should get back taskID 4 (since that is expected to complete // later than taskID 3). job.progressMade(taskAttemptID[3], 0.85f); job.progressMade(taskAttemptID[4], 0.20f); taskAttemptID[5] = job.findMapTask(trackers[4]); assertEquals(taskAttemptID[5].getTaskID().getId(), 4); // Verify total speculative tasks by jobtracker instrumentation assertEquals("Total speculative maps", 2, fakeInst.numSpeculativeMaps); assertEquals("Total speculative reduces", 3, fakeInst.numSpeculativeReduces); LOG.info("Total speculative maps = " + fakeInst.numSpeculativeMaps); LOG.info("Total speculative reduces = " + fakeInst.numSpeculativeReduces);
5445 -1009652281apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None TODO: figureout how to return length when there is no start and end SATD_ADDED getLength() public long getLength() throws IOException // TODO: figureout how to return length when there is no start and end return end - start;
5443 -1009652283apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None generate splits SATD_ADDED getSplits(JobContext) public List getSplits(JobContext job) throws IOException long minSize = Math.max(getFormatMinSplitSize(), getMinSplitSize(job)); long maxSize = getMaxSplitSize(job); // generate splits List splits = new ArrayList(); List files = listStatus(job); for (FileStatus file : files) { Path path = file.getPath(); long length = file.getLen(); if (length != 0) { FileSystem fs = path.getFileSystem(job.getConfiguration()); BlockLocation[] blkLocations = fs.getFileBlockLocations(file, 0, length); if (isSplitable(job, path)) { long blockSize = file.getBlockSize(); long splitSize = computeSplitSize(blockSize, minSize, maxSize); long bytesRemaining = length; while (((double) bytesRemaining) / splitSize > SPLIT_SLOP) { int blkIndex = getBlockIndex(blkLocations, length - bytesRemaining); splits.add(makeSplit(path, length - bytesRemaining, splitSize, blkLocations[blkIndex].getHosts())); bytesRemaining -= splitSize; } if (bytesRemaining != 0) { splits.add(makeSplit(path, length - bytesRemaining, bytesRemaining, blkLocations[blkLocations.length - 1].getHosts())); } } else { // not splitable splits.add(makeSplit(path, 0, length, blkLocations[0].getHosts())); } } else { // Create empty hosts array for zero length files splits.add(makeSplit(path, 0, length, new String[0])); } } // Save the number of input files for metrics/loadgen job.getConfiguration().setLong(NUM_INPUT_FILES, files.size()); LOG.debug("Total # of splits: " + splits.size()); return splits;
5442 -1009652284apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None TODO: move to a common with DataNode util class SATD_ADDED handshake(NamenodeProtocol) private static NamespaceInfo handshake(NamenodeProtocol namenode) throws IOException, SocketTimeoutException NamespaceInfo nsInfo; // throws SocketTimeoutException nsInfo = namenode.versionRequest(); String errorMsg = null; // verify build version if (!nsInfo.getBuildVersion().equals(Storage.getBuildVersion())) { errorMsg = "Incompatible build versions: active name-node BV = " + nsInfo.getBuildVersion() + "; backup node BV = " + Storage.getBuildVersion(); LOG.fatal(errorMsg); throw new IOException(errorMsg); } assert FSConstants.LAYOUT_VERSION == nsInfo.getLayoutVersion() : "Active and backup node layout versions must be the same. Expected: " + FSConstants.LAYOUT_VERSION + " actual " + nsInfo.getLayoutVersion(); return nsInfo;
5441 -1009652285apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None enable input decompression TODO replace with mapInputBytes and hdfsBytesRead SATD_ADDED configureCompressionEmulation(Configuration, Configuration) static void configureCompressionEmulation(Configuration source, Configuration target) // enable output compression target.setBoolean(FileOutputFormat.COMPRESS, source.getBoolean(FileOutputFormat.COMPRESS, false)); // set the job output compression codec String jobOutputCompressionCodec = source.get(FileOutputFormat.COMPRESS_CODEC); if (jobOutputCompressionCodec != null) { target.set(FileOutputFormat.COMPRESS_CODEC, jobOutputCompressionCodec); } // set the job output compression type String jobOutputCompressionType = source.get(FileOutputFormat.COMPRESS_TYPE); if (jobOutputCompressionType != null) { target.set(FileOutputFormat.COMPRESS_TYPE, jobOutputCompressionType); } // enable map output compression target.setBoolean(MRJobConfig.MAP_OUTPUT_COMPRESS, source.getBoolean(MRJobConfig.MAP_OUTPUT_COMPRESS, false)); // set the map output compression codecs String mapOutputCompressionCodec = source.get(MRJobConfig.MAP_OUTPUT_COMPRESS_CODEC); if (mapOutputCompressionCodec != null) { target.set(MRJobConfig.MAP_OUTPUT_COMPRESS_CODEC, mapOutputCompressionCodec); } // enable input decompression // TODO replace with mapInputBytes and hdfsBytesRead Path[] inputs = org.apache.hadoop.mapred.FileInputFormat.getInputPaths(new JobConf(source)); boolean needsCompressedInput = false; CompressionCodecFactory compressionCodecs = new CompressionCodecFactory(source); for (Path input : inputs) { CompressionCodec codec = compressionCodecs.getCodec(input); if (codec != null) { needsCompressedInput = true; } } setInputCompressionEmulationEnabled(target, needsCompressedInput);
5440 -1009652286apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None TODO Seek doesnt work with compressed input stream. Use SplittableCompressionCodec? SATD_ADDED getPossiblyDecompressedInputStream(Path, Configuration, long) static InputStream getPossiblyDecompressedInputStream(Path file, Configuration conf, long offset) throws IOException FileSystem fs = file.getFileSystem(conf); if (isCompressionEmulationEnabled(conf) && isInputCompressionEmulationEnabled(conf)) { CompressionCodecFactory compressionCodecs = new CompressionCodecFactory(conf); CompressionCodec codec = compressionCodecs.getCodec(file); if (codec != null) { Decompressor decompressor = CodecPool.getDecompressor(codec); if (decompressor != null) { CompressionInputStream in = codec.createInputStream(fs.open(file), decompressor); // TODO Seek doesnt work with compressed input stream. // Use SplittableCompressionCodec? return (InputStream) in; } } } FSDataInputStream in = fs.open(file); in.seek(offset); return (InputStream) in;
5439 -1009652287apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None TODO Control the extra data written .. TODO Should the key\tvalue\n be considered for measuring size? Can counters like BYTES_WRITTEN be used? What will be the value of such counters in LocalJobRunner? SATD_ADDED setup(Context) protected void setup(Context context) throws IOException, InterruptedException Configuration conf = context.getConfiguration(); int listSize = RandomTextDataGenerator.getRandomTextDataGeneratorListSize(conf); int wordSize = RandomTextDataGenerator.getRandomTextDataGeneratorWordSize(conf); rtg = new RandomTextDataGenerator(listSize, wordSize);
5438 -1009652288apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None TODO enable upload command only when selection is exactly one folder SATD_ADDED uploadDirectoryToDFS(IStructuredSelection) private void uploadDirectoryToDFS(IStructuredSelection selection) throws InvocationTargetException, InterruptedException // Ask the user which local directory to upload DirectoryDialog dialog = new DirectoryDialog(Display.getCurrent().getActiveShell(), SWT.OPEN | SWT.MULTI); dialog.setText("Select the local file or directory to upload"); String dirName = dialog.open(); final File dir = new File(dirName); List files = new ArrayList(); files.add(dir); // TODO enable upload command only when selection is exactly one folder final List folders = filterSelection(DFSFolder.class, selection); if (folders.size() >= 1) uploadToDFS(folders.get(0), files);
5437 -1009652289apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None TODO enable upload command only when selection is exactly one folder SATD_ADDED uploadFilesToDFS(IStructuredSelection) private void uploadFilesToDFS(IStructuredSelection selection) throws InvocationTargetException, InterruptedException // Ask the user which files to upload FileDialog dialog = new FileDialog(Display.getCurrent().getActiveShell(), SWT.OPEN | SWT.MULTI); dialog.setText("Select the local files to upload"); dialog.open(); List files = new ArrayList(); for (String fname : dialog.getFileNames()) files.add(new File(dialog.getFilterPath() + File.separator + fname)); // TODO enable upload command only when selection is exactly one folder List folders = filterSelection(DFSFolder.class, selection); if (folders.size() >= 1) uploadToDFS(folders.get(0), files);
5436 -1009652290apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None TODO SATD_ADDED init() public static void init() throws IOException GridmixTestUtils.initCluster();
5435 -1009652291apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None TODO Blocked by MAPREDUCE-118 SATD_ADDED init() public static void init() throws IOException GridmixTestUtils.initCluster();
5434 -1009652292apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None scan all policies once every 5 second SATD_ADDED mySetup(String, int) private void mySetup(String erasureCode, int rsParityLength) throws Exception // Make sure data directory exists new File(TEST_DIR).mkdirs(); conf = new Configuration(); conf.set("fs.raid.recoverylogdir", LOG_DIR); conf.setInt(RaidNode.RS_PARITY_LENGTH_KEY, rsParityLength); // scan all policies once every 5 second conf.setLong("raid.policy.rescan.interval", 5000); // make all deletions not go through Trash conf.set("fs.shell.delete.classname", "org.apache.hadoop.hdfs.DFSClient"); // do not use map-reduce cluster for Raiding conf.set("raid.classname", "org.apache.hadoop.raid.LocalRaidNode"); conf.set("raid.server.address", "localhost:0"); conf.setInt("hdfs.raid.stripeLength", stripeLength); conf.set("xor".equals(erasureCode) ? RaidNode.RAID_LOCATION_KEY : RaidNode.RAIDRS_LOCATION_KEY, "/destraid"); dfs = new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DATANODES).build(); dfs.waitActive(); fileSys = dfs.getFileSystem(); namenode = fileSys.getUri().toString(); hftp = "hftp://localhost.localdomain:" + dfs.getNameNodePort(); FileSystem.setDefaultUri(conf, namenode);
5433 -1009652293apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None abort execution, remove splits if nesc TODO release ThdLoc SATD_ADDED run() public void run() try { // pre-compute split information try { job.buildSplits(inputDir); } catch (IOException e) { LOG.warn("Failed to submit " + job.getJob().getJobName() + " as " + job.getUgi(), e); monitor.submissionFailed(job.getJob()); return; } catch (Exception e) { LOG.warn("Failed to submit " + job.getJob().getJobName() + " as " + job.getUgi(), e); monitor.submissionFailed(job.getJob()); return; } // Sleep until deadline long nsDelay = job.getDelay(TimeUnit.NANOSECONDS); while (nsDelay > 0) { TimeUnit.NANOSECONDS.sleep(nsDelay); nsDelay = job.getDelay(TimeUnit.NANOSECONDS); } try { // submit job monitor.add(job.call()); statistics.addJobStats(job.getJob(), job.getJobDesc()); LOG.debug("SUBMIT " + job + "@" + System.currentTimeMillis() + " (" + job.getJob().getJobID() + ")"); } catch (IOException e) { LOG.warn("Failed to submit " + job.getJob().getJobName() + " as " + job.getUgi(), e); if (e.getCause() instanceof ClosedByInterruptException) { throw new InterruptedException("Failed to submit " + job.getJob().getJobName()); } monitor.submissionFailed(job.getJob()); } catch (ClassNotFoundException e) { LOG.warn("Failed to submit " + job.getJob().getJobName(), e); monitor.submissionFailed(job.getJob()); } } catch (InterruptedException e) { // abort execution, remove splits if nesc // TODO release ThdLoc GridmixJob.pullDescription(job.id()); Thread.currentThread().interrupt(); monitor.submissionFailed(job.getJob()); } catch (Exception e) { // Due to some exception job wasnt submitted. LOG.info(" Job " + job.getJob() + " submission failed ", e); monitor.submissionFailed(job.getJob()); } finally { sem.release(); }
5432 -1009652294apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None pre-compute split information SATD_ADDED run() public void run() try { // pre-compute split information try { job.buildSplits(inputDir); } catch (IOException e) { LOG.warn("Failed to submit " + job.getJob().getJobName() + " as " + job.getUgi(), e); monitor.submissionFailed(job.getJob()); return; } catch (Exception e) { LOG.warn("Failed to submit " + job.getJob().getJobName() + " as " + job.getUgi(), e); monitor.submissionFailed(job.getJob()); return; } // Sleep until deadline long nsDelay = job.getDelay(TimeUnit.NANOSECONDS); while (nsDelay > 0) { TimeUnit.NANOSECONDS.sleep(nsDelay); nsDelay = job.getDelay(TimeUnit.NANOSECONDS); } try { // submit job monitor.add(job.call()); statistics.addJobStats(job.getJob(), job.getJobDesc()); LOG.debug("SUBMIT " + job + "@" + System.currentTimeMillis() + " (" + job.getJob().getJobID() + ")"); } catch (IOException e) { LOG.warn("Failed to submit " + job.getJob().getJobName() + " as " + job.getUgi(), e); if (e.getCause() instanceof ClosedByInterruptException) { throw new InterruptedException("Failed to submit " + job.getJob().getJobName()); } monitor.submissionFailed(job.getJob()); } catch (ClassNotFoundException e) { LOG.warn("Failed to submit " + job.getJob().getJobName(), e); monitor.submissionFailed(job.getJob()); } } catch (InterruptedException e) { // abort execution, remove splits if nesc // TODO release ThdLoc GridmixJob.pullDescription(job.id()); Thread.currentThread().interrupt(); monitor.submissionFailed(job.getJob()); } catch (Exception e) { // Due to some exception job wasnt submitted. LOG.info(" Job " + job.getJob() + " submission failed ", e); monitor.submissionFailed(job.getJob()); } finally { sem.release(); }
5431 -1009652295apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None explicitly do not use \"normal\" job.setOutputPath to make sure that it is not hardcoded anywhere in the framework. SATD_ADDED testWithLocal() public void testWithLocal() throws IOException, InterruptedException, ClassNotFoundException MiniMRCluster mr = null; try { mr = new MiniMRCluster(2, "file:///", 3); // make cleanup inline sothat validation of existence of these directories // can be done mr.setInlineCleanupThreads(); TestMiniMRWithDFS.runPI(mr, mr.createJobConf()); // run the wordcount example with caching JobConf job = mr.createJobConf(); TestResult ret = MRCaching.launchMRCache(TEST_ROOT_DIR + "/wc/input", TEST_ROOT_DIR + "/wc/output", TEST_ROOT_DIR + "/cachedir", job, "The quick brown fox\n" + "has many silly\n" + "red fox sox\n"); // assert the number of lines read during caching assertTrue("Failed test archives not matching", ret.isOutputOk); // test the task report fetchers JobClient client = new JobClient(job); JobID jobid = ret.job.getID(); TaskReport[] reports; reports = client.getSetupTaskReports(jobid); assertEquals("number of setups", 2, reports.length); reports = client.getMapTaskReports(jobid); assertEquals("number of maps", 1, reports.length); reports = client.getReduceTaskReports(jobid); assertEquals("number of reduces", 1, reports.length); reports = client.getCleanupTaskReports(jobid); assertEquals("number of cleanups", 2, reports.length); Counters counters = ret.job.getCounters(); assertEquals("number of map inputs", 3, counters.getCounter(TaskCounter.MAP_INPUT_RECORDS)); assertEquals("number of reduce outputs", 9, counters.getCounter(TaskCounter.REDUCE_OUTPUT_RECORDS)); runCustomFormats(mr); runSecondarySort(mr.createJobConf()); } finally { if (mr != null) { mr.shutdown(); } }
5430 -1009652296apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None OP_CLOSE 9 see above OP_SET_GENSTAMP 10 see above OP_SET_NS_QUOTA 11 obsolete OP_CLEAR_NS_QUOTA 12 obsolete OP_TIMES 13 Wed, 22 Sep 2010 22:45:27 GMT SATD_ADDED runOperations() private void runOperations() throws IOException LOG.info("Creating edits by performing fs operations"); // no check, if it's not it throws an exception which is what we want DistributedFileSystem dfs = (DistributedFileSystem) cluster.getFileSystem(); FileContext fc = FileContext.getFileContext(cluster.getURI(0), config); // OP_ADD 0, OP_SET_GENSTAMP 10 Path pathFileCreate = new Path("/file_create"); FSDataOutputStream s = dfs.create(pathFileCreate); // OP_CLOSE 9 s.close(); // OP_RENAME_OLD 1 Path pathFileMoved = new Path("/file_moved"); dfs.rename(pathFileCreate, pathFileMoved); // OP_DELETE 2 dfs.delete(pathFileMoved, false); // OP_MKDIR 3 Path pathDirectoryMkdir = new Path("/directory_mkdir"); dfs.mkdirs(pathDirectoryMkdir); // OP_SET_REPLICATION 4 s = dfs.create(pathFileCreate); s.close(); dfs.setReplication(pathFileCreate, (short) 1); // OP_SET_PERMISSIONS 7 Short permission = 0777; dfs.setPermission(pathFileCreate, new FsPermission(permission)); // OP_SET_OWNER 8 dfs.setOwner(pathFileCreate, new String("newOwner"), null); // OP_CLOSE 9 see above // OP_SET_GENSTAMP 10 see above // OP_SET_NS_QUOTA 11 obsolete // OP_CLEAR_NS_QUOTA 12 obsolete // OP_TIMES 13 // Wed, 22 Sep 2010 22:45:27 GMT long mtime = 1285195527000L; long atime = mtime; dfs.setTimes(pathFileCreate, mtime, atime); // OP_SET_QUOTA 14 dfs.setQuota(pathDirectoryMkdir, 1000L, FSConstants.QUOTA_DONT_SET); // OP_RENAME 15 fc.rename(pathFileCreate, pathFileMoved, Rename.NONE); // OP_CONCAT_DELETE 16 Path pathConcatTarget = new Path("/file_concat_target"); Path[] pathConcatFiles = new Path[2]; pathConcatFiles[0] = new Path("/file_concat_0"); pathConcatFiles[1] = new Path("/file_concat_1"); // multiple of blocksize for concat long length = blockSize * 3; short replication = 1; long seed = 1; DFSTestUtil.createFile(dfs, pathConcatTarget, length, replication, seed); DFSTestUtil.createFile(dfs, pathConcatFiles[0], length, replication, seed); DFSTestUtil.createFile(dfs, pathConcatFiles[1], length, replication, seed); dfs.concat(pathConcatTarget, pathConcatFiles); // OP_SYMLINK 17 Path pathSymlink = new Path("/file_symlink"); fc.createSymlink(pathConcatTarget, pathSymlink, false); // OP_GET_DELEGATION_TOKEN 18 final Token token = dfs.getDelegationToken("JobTracker"); // OP_RENEW_DELEGATION_TOKEN 19 // OP_CANCEL_DELEGATION_TOKEN 20 // see TestDelegationToken.java // fake the user to renew token for UserGroupInformation longUgi = UserGroupInformation.createRemoteUser("JobTracker/foo.com@FOO.COM"); UserGroupInformation shortUgi = UserGroupInformation.createRemoteUser("JobTracker"); try { longUgi.doAs(new PrivilegedExceptionAction() { public Object run() throws IOException { final DistributedFileSystem dfs = (DistributedFileSystem) cluster.getFileSystem(); dfs.renewDelegationToken(token); dfs.cancelDelegationToken(token); return null; } }); } catch (InterruptedException e) { throw new IOException("renewDelegationToken threw InterruptedException", e); } // OP_UPDATE_MASTER_KEY 21 // done by getDelegationTokenSecretManager().startThreads(); // sync to disk, otherwise we parse partial edits cluster.getNameNode().getFSImage().getEditLog().logSync(); // OP_REASSIGN_LEASE 22 String filePath = "/hard-lease-recovery-test"; byte[] bytes = "foo-bar-baz".getBytes(); DFSClientAdapter.stopLeaseRenewer(dfs.getClient()); FSDataOutputStream leaseRecoveryPath = dfs.create(new Path(filePath)); leaseRecoveryPath.write(bytes); leaseRecoveryPath.hflush(); // Set the hard lease timeout to 1 second. cluster.setLeasePeriod(60 * 1000, 1000); // wait for lease recovery to complete LocatedBlocks locatedBlocks; do { try { Thread.sleep(1000); } catch (InterruptedException e) { LOG.info("Innocuous exception", e); } locatedBlocks = DFSClientAdapter.callGetBlockLocations(cluster.getNameNode(), filePath, 0L, bytes.length); } while (locatedBlocks.isUnderConstruction());
5429 -1009652297apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None the env key is note present anywhere .. simply set it example X=$X:/tmp or X=/tmp SATD_ADDED getVMEnvironment(String, File, JobConf, Map, TaskAttemptID, long) private String getVMEnvironment(String errorInfo, File workDir, JobConf conf, Map env, TaskAttemptID taskid, long logSize) throws Throwable StringBuffer ldLibraryPath = new StringBuffer(); ldLibraryPath.append(workDir.toString()); String oldLdLibraryPath = null; oldLdLibraryPath = System.getenv("LD_LIBRARY_PATH"); if (oldLdLibraryPath != null) { ldLibraryPath.append(SYSTEM_PATH_SEPARATOR); ldLibraryPath.append(oldLdLibraryPath); } env.put("LD_LIBRARY_PATH", ldLibraryPath.toString()); // put jobTokenFile name into env String jobTokenFile = conf.get(TokenCache.JOB_TOKENS_FILENAME); LOG.debug("putting jobToken file name into environment fn=" + jobTokenFile); env.put(UserGroupInformation.HADOOP_TOKEN_FILE_LOCATION, jobTokenFile); // for the child of task jvm, set hadoop.root.logger env.put("HADOOP_ROOT_LOGGER", "INFO,TLA"); String hadoopClientOpts = System.getenv("HADOOP_CLIENT_OPTS"); if (hadoopClientOpts == null) { hadoopClientOpts = ""; } else { hadoopClientOpts = hadoopClientOpts + " "; } hadoopClientOpts = hadoopClientOpts + "-Dhadoop.tasklog.taskid=" + taskid + " -Dhadoop.tasklog.iscleanup=" + t.isTaskCleanupTask() + " -Dhadoop.tasklog.totalLogFileSize=" + logSize; env.put("HADOOP_CLIENT_OPTS", hadoopClientOpts); // add the env variables passed by the user String mapredChildEnv = getChildEnv(conf); if (mapredChildEnv != null && mapredChildEnv.length() > 0) { String[] childEnvs = mapredChildEnv.split(","); for (String cEnv : childEnvs) { try { // split on '=' String[] parts = cEnv.split("="); String value = env.get(parts[0]); if (value != null) { // replace $env with the child's env constructed by tt's // example LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/tmp value = parts[1].replace("$" + parts[0], value); } else { // this key is not configured by the tt for the child .. get it // from the tt's env // example PATH=$PATH:/tmp value = System.getenv(parts[0]); if (value != null) { // the env key is present in the tt's env value = parts[1].replace("$" + parts[0], value); } else { // the env key is note present anywhere .. simply set it // example X=$X:/tmp or X=/tmp value = parts[1].replace("$" + parts[0], ""); } } env.put(parts[0], value); } catch (Throwable t) { // set the error msg errorInfo = "Invalid User environment settings : " + mapredChildEnv + ". Failed to parse user-passed environment param." + " Expecting : env1=value1,env2=value2..."; LOG.warn(errorInfo); throw t; } } } return errorInfo;
5428 -1009652298apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Delegate the generation of input splits to the 'original' InputFormat SATD_ADDED NullWritable> getRecordReader(InputSplit, JobConf, Reporter) public RecordReader getRecordReader(InputSplit genericSplit, JobConf job, Reporter reporter) throws IOException return new PipesDummyRecordReader(job, genericSplit);
5427 -1009652299apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None if remainder is between max and 2*max - then instead of creating splits of size max, left-max we create splits of size left/2 and left/2. This is a heuristic to avoid creating really really small splits. SATD_ADDED setMaxSplitSize(long) protected void setMaxSplitSize(long maxSplitSize) this.maxSplitSize = maxSplitSize;
5426 -1009652300apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None if there is a minimum size specified, then create a single split otherwise, store these blocks into overflow data structure SATD_ADDED setMaxSplitSize(long) protected void setMaxSplitSize(long maxSplitSize) this.maxSplitSize = maxSplitSize;
5425 -1009652301apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Create one split for this rack before moving over to the next rack. Come back to this rack after creating a single split for each of the remaining racks. Process one rack location at a time, Combine all possible blocks that reside on this rack as one split. (constrained by minimum and maximum split size). SATD_ADDED setMaxSplitSize(long) protected void setMaxSplitSize(long maxSplitSize) this.maxSplitSize = maxSplitSize;
5423 -1009652303apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None TODO: test will fail if the following is un-commented. SATD_ADDED checkFileStatus(FileStatus, FileStatus) private static void checkFileStatus(final FileStatus expected, final FileStatus computed) Assert.assertEquals(expected.getPath().toUri().getPath(), computed.getPath().toUri().getPath()); // TODO: test will fail if the following is un-commented. // Assert.assertEquals(expected.getAccessTime(), computed.getAccessTime()); // Assert.assertEquals(expected.getModificationTime(), // computed.getModificationTime()); Assert.assertEquals(expected.getBlockSize(), computed.getBlockSize()); Assert.assertEquals(expected.getGroup(), computed.getGroup()); Assert.assertEquals(expected.getLen(), computed.getLen()); Assert.assertEquals(expected.getOwner(), computed.getOwner()); Assert.assertEquals(expected.getPermission(), computed.getPermission()); Assert.assertEquals(expected.getReplication(), computed.getReplication());
5422 -1009652304apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None how much of the task time belonged to current interval SATD_ADDED toString() public String toString() return statName;
5421 -1009652305apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Sort the racks based on their contribution to this split SATD_ADDED accept(Path) public boolean accept(Path p) String name = p.getName(); return !name.startsWith("_") && !name.startsWith(".");
5420 -1009652306apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None generate splits SATD_ADDED accept(Path) public boolean accept(Path p) String name = p.getName(); return !name.startsWith("_") && !name.startsWith(".");
5419 -1009652307apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Hack for local FS that does not have the concept of a 'mounting point' SATD_ADDED getFlagDir(boolean) private static Path getFlagDir(boolean local) Path flagDir = new Path("testing/chain/flags"); // Hack for local FS that does not have the concept of a 'mounting point' if (local) { String localPathRoot = System.getProperty("test.build.data", "/tmp").replace(' ', '+'); flagDir = new Path(localPathRoot, flagDir); } return flagDir;
5417 -1009652309apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None for security authorization server principal for this call should be NAMENODE's one. SATD_ADDED refreshSuperUserGroupsConfiguration() public int refreshSuperUserGroupsConfiguration() throws IOException // Get the current configuration Configuration conf = getConf(); // for security authorization // server principal for this call // should be NAMENODE's one. conf.set(CommonConfigurationKeys.HADOOP_SECURITY_SERVICE_USER_NAME_KEY, conf.get(DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY, "")); // Create the client RefreshUserMappingsProtocol refreshProtocol = (RefreshUserMappingsProtocol) RPC.getProxy(RefreshUserMappingsProtocol.class, RefreshUserMappingsProtocol.versionID, NameNode.getAddress(conf), getUGI(), conf, NetUtils.getSocketFactory(conf, RefreshUserMappingsProtocol.class)); // Refresh the user-to-groups mappings refreshProtocol.refreshSuperUserGroupsConfiguration(); return 0;
5416 -1009652310apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None for security authorization server principal for this call should be NN's one. SATD_ADDED refreshUserToGroupsMappings() public int refreshUserToGroupsMappings() throws IOException // Get the current configuration Configuration conf = getConf(); // for security authorization // server principal for this call // should be NN's one. conf.set(CommonConfigurationKeys.HADOOP_SECURITY_SERVICE_USER_NAME_KEY, conf.get(DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY, "")); // Create the client RefreshUserMappingsProtocol refreshProtocol = (RefreshUserMappingsProtocol) RPC.getProxy(RefreshUserMappingsProtocol.class, RefreshUserMappingsProtocol.versionID, NameNode.getAddress(conf), getUGI(), conf, NetUtils.getSocketFactory(conf, RefreshUserMappingsProtocol.class)); // Refresh the user-to-groups mappings refreshProtocol.refreshUserToGroupsMappings(); return 0;
5415 -1009652311apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Refresh the authorization policy in-effect SATD_ADDED refreshServiceAcl() public int refreshServiceAcl() throws IOException // Get the current configuration Configuration conf = getConf(); // for security authorization // server principal for this call // should be NN's one. conf.set(CommonConfigurationKeys.HADOOP_SECURITY_SERVICE_USER_NAME_KEY, conf.get(DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY, "")); // Create the client RefreshAuthorizationPolicyProtocol refreshProtocol = (RefreshAuthorizationPolicyProtocol) RPC.getProxy(RefreshAuthorizationPolicyProtocol.class, RefreshAuthorizationPolicyProtocol.versionID, NameNode.getAddress(conf), getUGI(), conf, NetUtils.getSocketFactory(conf, RefreshAuthorizationPolicyProtocol.class)); // Refresh the authorization policy in-effect refreshProtocol.refreshServiceAcl(); return 0;
5414 -1009652312apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None for security authorization server principal for this call should be NN's one. SATD_ADDED refreshServiceAcl() public int refreshServiceAcl() throws IOException // Get the current configuration Configuration conf = getConf(); // for security authorization // server principal for this call // should be NN's one. conf.set(CommonConfigurationKeys.HADOOP_SECURITY_SERVICE_USER_NAME_KEY, conf.get(DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY, "")); // Create the client RefreshAuthorizationPolicyProtocol refreshProtocol = (RefreshAuthorizationPolicyProtocol) RPC.getProxy(RefreshAuthorizationPolicyProtocol.class, RefreshAuthorizationPolicyProtocol.versionID, NameNode.getAddress(conf), getUGI(), conf, NetUtils.getSocketFactory(conf, RefreshAuthorizationPolicyProtocol.class)); // Refresh the authorization policy in-effect refreshProtocol.refreshServiceAcl(); return 0;
5413 -1009652313apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None it's fine walking down the entire list of running jobs - there probably will not be many, plus, we may need to go through the list to compute numSlotsOccupiedByUser. If this is expensive, we can keep a list of running jobs per user. Then we only need to consider the first few jobs per user. SATD_ADDED compare(JobSchedulingInfo, JobSchedulingInfo) public int compare(JobSchedulingInfo o1, JobSchedulingInfo o2) // the job that started earlier wins if (o1.getStartTime() < o2.getStartTime()) { return -1; } else { return (o1.getStartTime() == o2.getStartTime() ? o1.getJobID().compareTo(o2.getJobID()) : 1); }
5412 -1009652314apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None TODO: We have just 2 types of listeners as of now . If no of listeners increase then we should move to map kind of model. SATD_ADDED run() public JobClient run() throws IOException return new JobClient(new JobConf(conf));
5411 -1009652315apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None This kind of error doesn't mean that the stream itself is broken - just the flushing thread got interrupted. So, we shouldn't close down the writer, but instead just propagate the error SATD_ADDED hflush() public void hflush() throws IOException dfsClient.checkOpen(); isClosed(); try { long toWaitFor; synchronized (this) { /* Record current blockOffset. This might be changed inside * flushBuffer() where a partial checksum chunk might be flushed. * After the flush, reset the bytesCurBlock back to its previous value, * any partial checksum chunk will be sent now and in next packet. */ long saveOffset = bytesCurBlock; Packet oldCurrentPacket = currentPacket; // flush checksum buffer, but keep checksum buffer intact flushBuffer(true); // bytesCurBlock potentially incremented if there was buffered data if (DFSClient.LOG.isDebugEnabled()) { DFSClient.LOG.debug("DFSClient flush() : saveOffset " + saveOffset + " bytesCurBlock " + bytesCurBlock + " lastFlushOffset " + lastFlushOffset); } // Flush only if we haven't already flushed till this offset. if (lastFlushOffset != bytesCurBlock) { assert bytesCurBlock > lastFlushOffset; // record the valid offset of this flush lastFlushOffset = bytesCurBlock; waitAndQueueCurrentPacket(); } else { // We already flushed up to this offset. // This means that we haven't written anything since the last flush // (or the beginning of the file). Hence, we should not have any // packet queued prior to this call, since the last flush set // currentPacket = null. assert oldCurrentPacket == null : "Empty flush should not occur with a currentPacket"; // just discard the current packet since it is already been sent. currentPacket = null; } // Restore state of stream. Record the last flush offset // of the last full chunk that was flushed. // bytesCurBlock = saveOffset; toWaitFor = lastQueuedSeqno; } // end synchronized waitForAckedSeqno(toWaitFor); // If any new blocks were allocated since the last flush, // then persist block locations on namenode. // if (persistBlocks.getAndSet(false)) { try { dfsClient.namenode.fsync(src, dfsClient.clientName); } catch (IOException ioe) { DFSClient.LOG.warn("Unable to persist blocks in hflush for " + src, ioe); // If we got an error here, it might be because some other thread called // close before our hflush completed. In that case, we should throw an // exception that the stream is closed. isClosed(); // If we aren't closed but failed to sync, we should expose that to the // caller. throw ioe; } } synchronized (this) { if (streamer != null) { streamer.setHflush(); } } } catch (InterruptedIOException interrupt) { // This kind of error doesn't mean that the stream itself is broken - just the // flushing thread got interrupted. So, we shouldn't close down the writer, // but instead just propagate the error throw interrupt; } catch (IOException e) { DFSClient.LOG.warn("Error while syncing", e); synchronized (this) { if (!closed) { lastException = new IOException("IOException flush:" + e); closeThreads(true); } } throw e; }
5410 -1009652316apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None setup pipeline to append to the last block XXX retries?? SATD_ADDED writeData(byte[], int, int) void writeData(byte[] inarray, int off, int len) if (dataPos + len > buf.length) { throw new BufferOverflowException(); } System.arraycopy(inarray, off, buf, dataPos, len); dataPos += len;
5408 -1009652318apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None check if a block is marked as moved SATD_ADDED chooseBlockAndProxy() private boolean chooseBlockAndProxy() // iterate all source's blocks until find a good one for (Iterator blocks = source.getBlockIterator(); blocks.hasNext(); ) { if (markMovedIfGoodBlock(blocks.next())) { blocks.remove(); return true; } } return false;
5407 -1009652319apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None add a block thus marking a block to be moved SATD_ADDED chooseBlockAndProxy() private boolean chooseBlockAndProxy() // iterate all source's blocks until find a good one for (Iterator blocks = source.getBlockIterator(); blocks.hasNext(); ) { if (markMovedIfGoodBlock(blocks.next())) { blocks.remove(); return true; } } return false;
5406 -1009652320apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None initialize the moved blocks collection SATD_ADDED chooseBlockAndProxy() private boolean chooseBlockAndProxy() // iterate all source's blocks until find a good one for (Iterator blocks = source.getBlockIterator(); blocks.hasNext(); ) { if (markMovedIfGoodBlock(blocks.next())) { blocks.remove(); return true; } } return false;
5405 -1009652321apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None The sleeping period before checking if block move is completed again SATD_ADDED chooseBlockAndProxy() private boolean chooseBlockAndProxy() // iterate all source's blocks until find a good one for (Iterator blocks = source.getBlockIterator(); blocks.hasNext(); ) { if (markMovedIfGoodBlock(blocks.next())) { blocks.remove(); return true; } } return false;
5404 -1009652322apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Start a thread to dispatch block moves for each source. The thread selects blocks to move & sends request to proxy source to initiate block move. The process is flow controlled. Block selection is blocked if there are too many un-confirmed block moves. Return the total number of bytes successfully moved in this iteration. SATD_ADDED chooseBlockAndProxy() private boolean chooseBlockAndProxy() // iterate all source's blocks until find a good one for (Iterator blocks = source.getBlockIterator(); blocks.hasNext(); ) { if (markMovedIfGoodBlock(blocks.next())) { blocks.remove(); return true; } } return false;
5403 -1009652323apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Decide all pairs and the number of bytes to move from a source to a target Maximum bytes to be moved per node is Min(1 Band worth of bytes, MAX_SIZE_TO_MOVE). Return total number of bytes to move in this iteration SATD_ADDED chooseBlockAndProxy() private boolean chooseBlockAndProxy() // iterate all source's blocks until find a good one for (Iterator blocks = source.getBlockIterator(); blocks.hasNext(); ) { if (markMovedIfGoodBlock(blocks.next())) { blocks.remove(); return true; } } return false;
5402 -1009652324apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None filter already moved blocks SATD_ADDED chooseBlockAndProxy() private boolean chooseBlockAndProxy() // iterate all source's blocks until find a good one for (Iterator blocks = source.getBlockIterator(); blocks.hasNext(); ) { if (markMovedIfGoodBlock(blocks.next())) { blocks.remove(); return true; } } return false;
5401 -1009652325apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None move the block SATD_ADDED chooseBlockAndProxy() private boolean chooseBlockAndProxy() // iterate all source's blocks until find a good one for (Iterator blocks = source.getBlockIterator(); blocks.hasNext(); ) { if (markMovedIfGoodBlock(blocks.next())) { blocks.remove(); return true; } } return false;
5400 -1009652326apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None This method iteratively does the following: it first selects a block to move, then sends a request to the proxy source to start the block move when the source's block list falls below a threshold, it asks the namenode for more blocks. It terminates when it has dispatch enough block move tasks or it has received enough blocks from the namenode, or the elapsed time of the iteration has exceeded the max time limit. SATD_ADDED chooseBlockAndProxy() private boolean chooseBlockAndProxy() // iterate all source's blocks until find a good one for (Iterator blocks = source.getBlockIterator(); blocks.hasNext(); ) { if (markMovedIfGoodBlock(blocks.next())) { blocks.remove(); return true; } } return false;
5399 -1009652327apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None iterate all source's blocks to remove moved ones SATD_ADDED chooseBlockAndProxy() private boolean chooseBlockAndProxy() // iterate all source's blocks until find a good one for (Iterator blocks = source.getBlockIterator(); blocks.hasNext(); ) { if (markMovedIfGoodBlock(blocks.next())) { blocks.remove(); return true; } } return false;
5398 -1009652328apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None cancel the tentative move SATD_ADDED chooseBlockAndProxy() private boolean chooseBlockAndProxy() // iterate all source's blocks until find a good one for (Iterator blocks = source.getBlockIterator(); blocks.hasNext(); ) { if (markMovedIfGoodBlock(blocks.next())) { blocks.remove(); return true; } } return false;
5397 -1009652329apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Return a block that's good for the source thread to dispatch immediately The block's source, target, and proxy source are determined too. When choosing proxy and target, source & target throttling has been considered. They are chosen only when they have the capacity to support this block move. The block should be dispatched immediately after this method is returned. SATD_ADDED chooseBlockAndProxy() private boolean chooseBlockAndProxy() // iterate all source's blocks until find a good one for (Iterator blocks = source.getBlockIterator(); blocks.hasNext(); ) { if (markMovedIfGoodBlock(blocks.next())) { blocks.remove(); return true; } } return false;
5396 -1009652330apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Decide if the given block is a good candidate to move or not SATD_ADDED chooseBlockAndProxy() private boolean chooseBlockAndProxy() // iterate all source's blocks until find a good one for (Iterator blocks = source.getBlockIterator(); blocks.hasNext(); ) { if (markMovedIfGoodBlock(blocks.next())) { blocks.remove(); return true; } } return false;
5395 -1009652331apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None A thread that initiates a block move and waits for block move to complete SATD_ADDED chooseBlockAndProxy() private boolean chooseBlockAndProxy() // iterate all source's blocks until find a good one for (Iterator blocks = source.getBlockIterator(); blocks.hasNext(); ) { if (markMovedIfGoodBlock(blocks.next())) { blocks.remove(); return true; } } return false;
5394 -1009652332apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Add a scheduled block move to the node SATD_ADDED chooseBlockAndProxy() private boolean chooseBlockAndProxy() // iterate all source's blocks until find a good one for (Iterator blocks = source.getBlockIterator(); blocks.hasNext(); ) { if (markMovedIfGoodBlock(blocks.next())) { blocks.remove(); return true; } } return false;
5393 -1009652333apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None start a thread to dispatch the block move SATD_ADDED chooseBlockAndProxy() private boolean chooseBlockAndProxy() // iterate all source's blocks until find a good one for (Iterator blocks = source.getBlockIterator(); blocks.hasNext(); ) { if (markMovedIfGoodBlock(blocks.next())) { blocks.remove(); return true; } } return false;
5392 -1009652334apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Dispatch the block move task to the proxy source & wait for the response SATD_ADDED chooseBlockAndProxy() private boolean chooseBlockAndProxy() // iterate all source's blocks until find a good one for (Iterator blocks = source.getBlockIterator(); blocks.hasNext(); ) { if (markMovedIfGoodBlock(blocks.next())) { blocks.remove(); return true; } } return false;
5391 -1009652335apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None What to include in the published price in spending per slot SATD_ADDED addBudget(String, float) void addBudget(String queue, float budget) store.addBudget(queue, budget);
5390 -1009652336apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None see if a different TIP might work better. SATD_ADDED getTimestamp() public long getTimestamp() return timestamp;
5389 -1009652337apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Ugly! Convert the trackerName to it's host name SATD_ADDED getTimestamp() public long getTimestamp() return timestamp;
5388 -1009652338apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None TODO The only problem with these counters would be on restart. The jobtracker updates the counter only when the task that is scheduled if from a non-running tip and is local (data, rack ...). But upon restart as the reports come from the task tracker, there is no good way to infer when exactly to increment the locality counters. The only solution is to increment the counters for all the tasks irrespective of - whether the tip is running or not - whether its a speculative task or not So to simplify, increment the data locality counter whenever there is data locality. SATD_ADDED getTimestamp() public long getTimestamp() return timestamp;
5386 -1009652340apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None do the clean up of temporary directory SATD_ADDED cleanupJob(JobContext) public void cleanupJob(JobContext context) throws IOException JobConf conf = context.getJobConf(); // do the clean up of temporary directory Path outputPath = FileOutputFormat.getOutputPath(conf); if (outputPath != null) { Path tmpDir = new Path(outputPath, FileOutputCommitter.TEMP_DIR_NAME); FileSystem fileSys = tmpDir.getFileSystem(conf); context.getProgressible().progress(); if (fileSys.exists(tmpDir)) { fileSys.delete(tmpDir, true); } else { LOG.warn("Output Path is Null in cleanup"); } }
5385 -1009652341apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None TODO Remove this once LocalJobRunner can run Gridmix. SATD_ADDED getSplits(JobContext) public List getSplits(JobContext jobCtxt) throws IOException // get the total data to be generated long toGen = jobCtxt.getConfiguration().getLong(GenerateData.GRIDMIX_GEN_BYTES, -1); if (toGen < 0) { throw new IOException("Invalid/missing generation bytes: " + toGen); } // get the total number of mappers configured int totalMappersConfigured = jobCtxt.getConfiguration().getInt(MRJobConfig.NUM_MAPS, -1); if (totalMappersConfigured < 0) { throw new IOException("Invalid/missing num mappers: " + totalMappersConfigured); } final long bytesPerTracker = toGen / totalMappersConfigured; final ArrayList splits = new ArrayList(totalMappersConfigured); for (int i = 0; i < totalMappersConfigured; ++i) { splits.add(new GenSplit(bytesPerTracker, new String[] { "tracker_local" })); } return splits;
5384 -1009652342apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None This is a hack to pass the sleep duration via Gridmix key TODO: We need to come up with better solution for this. SATD_ADDED initialValue() protected Random initialValue() return new Random();
5383 -1009652343apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None rare case where splits are exact, logs.length can be 4 SATD_ADDED reset() void reset() final int oldsize = size; do { size = gen.nextInt(MAX_SIZE); } while (oldsize == size); final long oldseed = seed; do { seed = gen.nextLong() & Long.MAX_VALUE; } while (oldseed == seed);
5382 -1009652344apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None returns an iterator of all blocks in a given priority queue SATD_ADDED clear() void clear() for (int i = 0; i < LEVEL; i++) { priorityQueues.get(i).clear(); }
5381 -1009652345apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None add a block to a under replication queue according to its priority @param block a under replication block @param curReplicas current number of replicas of the block @param expectedReplicas expected number of replicas of the block SATD_ADDED clear() void clear() for (int i = 0; i < LEVEL; i++) { priorityQueues.get(i).clear(); }
5380 -1009652346apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Return the priority of a block @param block a under replication block @param curReplicas current number of replicas of the block @param expectedReplicas expected number of replicas of the block SATD_ADDED clear() void clear() for (int i = 0; i < LEVEL; i++) { priorityQueues.get(i).clear(); }
5379 -1009652347apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Don't check for exact values in the middle, because the splitter generates some ugly Unicode-isms. But do check that we get multiple splits and that it starts and ends on the correct points. SATD_ADDED testCommonPrefix() public void testCommonPrefix() throws SQLException // Splits between 'Hand' and 'Hardy' TextSplitter splitter = new TextSplitter(); List splits = splitter.split(5, "nd", "rdy", "Ha"); // Don't check for exact values in the middle, because the splitter generates some // ugly Unicode-isms. But do check that we get multiple splits and that it starts // and ends on the correct points. assertEquals("Hand", splits.get(0)); assertEquals("Hardy", splits.get(splits.size() - 1)); assertEquals(6, splits.size());
5378 -1009652348apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None This should give us 25 splits, one per letter. SATD_ADDED testAlphabetSplit() public void testAlphabetSplit() throws SQLException // This should give us 25 splits, one per letter. TextSplitter splitter = new TextSplitter(); List splits = splitter.split(25, "A", "Z", ""); String[] expected = { "A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z" }; assertArrayEquals(expected, splits.toArray(new String[0]));
5377 -1009652349apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None the block is only in one of the to-do lists if it is in none then data-node already has it SATD_ADDED addBlock(DatanodeDescriptor, Block, String) void addBlock(DatanodeDescriptor node, Block block, String delHint) throws IOException // decrement number of blocks scheduled to this datanode. node.decBlocksScheduled(); // get the deletion hint node DatanodeDescriptor delHintNode = null; if (delHint != null && delHint.length() != 0) { delHintNode = namesystem.getDatanode(delHint); if (delHintNode == null) { NameNode.stateChangeLog.warn("BLOCK* NameSystem.blockReceived: " + block + " is expected to be removed from an unrecorded node " + delHint); } } // // Modify the blocks->datanode map and node's map. // pendingReplications.remove(block); // blockReceived reports a finalized block Collection toAdd = new LinkedList(); Collection toInvalidate = new LinkedList(); Collection toCorrupt = new LinkedList(); Collection toUC = new LinkedList(); processReportedBlock(node, block, ReplicaState.FINALIZED, toAdd, toInvalidate, toCorrupt, toUC); // the block is only in one of the to-do lists // if it is in none then data-node already has it assert toUC.size() + toAdd.size() + toInvalidate.size() + toCorrupt.size() <= 1 : "The block should be only in one of the lists."; for (StatefulBlockInfo b : toUC) { addStoredBlockUnderConstruction(b.storedBlock, node, b.reportedState); } for (BlockInfo b : toAdd) { addStoredBlock(b, node, delHintNode, true); } for (Block b : toInvalidate) { NameNode.stateChangeLog.info("BLOCK* NameSystem.addBlock: block " + b + " on " + node.getName() + " size " + b.getNumBytes() + " does not belong to any file."); addToInvalidates(b, node); } for (BlockInfo b : toCorrupt) { markBlockAsCorrupt(b, node); }
5376 -1009652350apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None The next two methods test the various cases under which we must conclude the replica is corrupt, or under construction. These are laid out as switch statements, on the theory that it is easier to understand the combinatorics of reportedState and ucState that way. It should be at least as efficient as boolean expressions. SATD_ADDED isReplicaCorrupt(Block, ReplicaState, BlockInfo, BlockUCState, DatanodeDescriptor) private boolean isReplicaCorrupt(Block iblk, ReplicaState reportedState, BlockInfo storedBlock, BlockUCState ucState, DatanodeDescriptor dn) switch(reportedState) { case FINALIZED: switch(ucState) { case COMPLETE: case COMMITTED: return (storedBlock.getGenerationStamp() != iblk.getGenerationStamp() || storedBlock.getNumBytes() != iblk.getNumBytes()); default: return false; } case RBW: case RWR: return storedBlock.isComplete(); // should not be reported case RUR: // should not be reported case TEMPORARY: default: FSNamesystem.LOG.warn("Unexpected replica state " + reportedState + " for block: " + storedBlock + " on " + dn.getName() + " size " + storedBlock.getNumBytes()); return true; }
5375 -1009652351apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None move block to the head of the list SATD_ADDED reportDiff(DatanodeDescriptor, BlockListAsLongs, Collection, Collection, Collection, Collection, Collection) void reportDiff(DatanodeDescriptor dn, BlockListAsLongs newReport, Collection toAdd, Collection toRemove, Collection toInvalidate, Collection toCorrupt, Collection toUC) // add to under-construction list // place a delimiter in the list which separates blocks // that have been reported from those that have not BlockInfo delimiter = new BlockInfo(new Block(), 1); boolean added = dn.addBlock(delimiter); assert added : "Delimiting block cannot be present in the node"; if (newReport == null) newReport = new BlockListAsLongs(); // scan the report and process newly reported blocks BlockReportIterator itBR = newReport.getBlockReportIterator(); while (itBR.hasNext()) { Block iblk = itBR.next(); ReplicaState iState = itBR.getCurrentReplicaState(); BlockInfo storedBlock = processReportedBlock(dn, iblk, iState, toAdd, toInvalidate, toCorrupt, toUC); // move block to the head of the list if (storedBlock != null && storedBlock.findDatanode(dn) >= 0) dn.moveBlockToHead(storedBlock); } // collect blocks that have not been reported // all of them are next to the delimiter Iterator it = new DatanodeDescriptor.BlockIterator(delimiter.getNext(0), dn); while (it.hasNext()) toRemove.add(it.next()); dn.removeBlock(delimiter);
5374 -1009652352apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Initial block reports can be processed a lot more efficiently than ordinary block reports. This shortens NN restart times. SATD_ADDED processReport(DatanodeDescriptor, BlockListAsLongs) public void processReport(DatanodeDescriptor node, BlockListAsLongs report) throws IOException boolean isFirstBlockReport = (node.numBlocks() == 0); if (isFirstBlockReport) { // Initial block reports can be processed a lot more efficiently than // ordinary block reports. This shortens NN restart times. processFirstBlockReport(node, report); return; } // Normal case: // Modify the (block-->datanode) map, according to the difference // between the old and new block report. // Collection toAdd = new LinkedList(); Collection toRemove = new LinkedList(); Collection toInvalidate = new LinkedList(); Collection toCorrupt = new LinkedList(); Collection toUC = new LinkedList(); reportDiff(node, report, toAdd, toRemove, toInvalidate, toCorrupt, toUC); // Process the blocks on each queue for (StatefulBlockInfo b : toUC) { addStoredBlockUnderConstruction(b.storedBlock, node, b.reportedState); } for (Block b : toRemove) { removeStoredBlock(b, node); } for (BlockInfo b : toAdd) { addStoredBlock(b, node, null, true); } for (Block b : toInvalidate) { NameNode.stateChangeLog.info("BLOCK* NameSystem.processReport: block " + b + " on " + node.getName() + " size " + b.getNumBytes() + " does not belong to any file."); addToInvalidates(b, node); } for (BlockInfo b : toCorrupt) { markBlockAsCorrupt(b, node); }
5373 -1009652353apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None TODO should using recentInvalidateSets be synchronized? get an array of the keys SATD_ADDED setConfigurationParameters(Configuration) void setConfigurationParameters(Configuration conf) throws IOException this.replicator = BlockPlacementPolicy.getInstance(conf, namesystem, namesystem.clusterMap); this.maxCorruptFilesReturned = conf.getInt(DFSConfigKeys.DFS_DEFAULT_MAX_CORRUPT_FILES_RETURNED_KEY, DFSConfigKeys.DFS_DEFAULT_MAX_CORRUPT_FILES_RETURNED); this.defaultReplication = conf.getInt(DFSConfigKeys.DFS_REPLICATION_KEY, DFSConfigKeys.DFS_REPLICATION_DEFAULT); this.maxReplication = conf.getInt(DFSConfigKeys.DFS_REPLICATION_MAX_KEY, DFSConfigKeys.DFS_REPLICATION_MAX_DEFAULT); this.minReplication = conf.getInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MIN_KEY, DFSConfigKeys.DFS_NAMENODE_REPLICATION_MIN_DEFAULT); if (minReplication <= 0) throw new IOException("Unexpected configuration parameters: dfs.namenode.replication.min = " + minReplication + " must be greater than 0"); if (maxReplication >= (int) Short.MAX_VALUE) throw new IOException("Unexpected configuration parameters: dfs.replication.max = " + maxReplication + " must be less than " + (Short.MAX_VALUE)); if (maxReplication < minReplication) throw new IOException("Unexpected configuration parameters: dfs.namenode.replication.min = " + minReplication + " must be less than dfs.replication.max = " + maxReplication); this.maxReplicationStreams = conf.getInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_DEFAULT); this.shouldCheckForEnoughRacks = conf.get(DFSConfigKeys.NET_TOPOLOGY_SCRIPT_FILE_NAME_KEY) == null ? false : true; FSNamesystem.LOG.info("defaultReplication = " + defaultReplication); FSNamesystem.LOG.info("maxReplication = " + maxReplication); FSNamesystem.LOG.info("minReplication = " + minReplication); FSNamesystem.LOG.info("maxReplicationStreams = " + maxReplicationStreams); FSNamesystem.LOG.info("shouldCheckForEnoughRacks = " + shouldCheckForEnoughRacks);
5371 -1009652355apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None TODO: ensure that no jobs/tasks are running restart the cluster if cleanup fails SATD_ADDED ensureClean() public void ensureClean() throws IOException // TODO: ensure that no jobs/tasks are running // restart the cluster if cleanup fails JTClient jtClient = getJTClient(); JobInfo[] jobs = jtClient.getProxy().getAllJobInfo(); for (JobInfo job : jobs) { jtClient.killJob(org.apache.hadoop.mapred.JobID.downgrade(job.getID())); }
5370 -1009652356apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Now we need to generate the random numbers according to the above distribution. We create a lot of map tasks, each of which takes at least one \"line\" of the distribution. (That is, a certain number X is to be generated Y number of times.) A map task emits Y key/val pairs. The val is X. The key is a randomly-generated number. The reduce task gets its input sorted by key. That is, sorted in random order. It then emits a single line of text that for the given values. It does not emit the key. Because there's just one reduce task, we emit a single big file of random numbers. SATD_ADDED launch() private static void launch() throws Exception // // Generate distribution of ints. This is the answer key. // Configuration conf = new Configuration(); int countsToGo = counts; int[] dist = new int[range]; for (int i = 0; i < range; i++) { double avgInts = (1.0 * countsToGo) / (range - i); dist[i] = (int) Math.max(0, Math.round(avgInts + (Math.sqrt(avgInts) * r.nextGaussian()))); countsToGo -= dist[i]; } if (countsToGo > 0) { dist[dist.length - 1] += countsToGo; } // // Write the answer key to a file. // Path testdir = new Path("mapred.loadtest"); if (!fs.mkdirs(testdir)) { throw new IOException("Mkdirs failed to create " + testdir.toString()); } Path randomIns = new Path(testdir, "genins"); if (!fs.mkdirs(randomIns)) { throw new IOException("Mkdirs failed to create " + randomIns.toString()); } Path answerkey = new Path(randomIns, "answer.key"); SequenceFile.Writer out = SequenceFile.createWriter(fs, conf, answerkey, IntWritable.class, IntWritable.class, SequenceFile.CompressionType.NONE); try { for (int i = 0; i < range; i++) { out.append(new IntWritable(i), new IntWritable(dist[i])); } } finally { out.close(); } printFiles(randomIns, conf); // // Now we need to generate the random numbers according to // the above distribution. // // We create a lot of map tasks, each of which takes at least // one "line" of the distribution. (That is, a certain number // X is to be generated Y number of times.) // // A map task emits Y key/val pairs. The val is X. The key // is a randomly-generated number. // // The reduce task gets its input sorted by key. That is, sorted // in random order. It then emits a single line of text that // for the given values. It does not emit the key. // // Because there's just one reduce task, we emit a single big // file of random numbers. // Path randomOuts = new Path(testdir, "genouts"); fs.delete(randomOuts, true); Job genJob = Job.getInstance(conf); FileInputFormat.setInputPaths(genJob, randomIns); genJob.setInputFormatClass(SequenceFileInputFormat.class); genJob.setMapperClass(RandomGenMapper.class); FileOutputFormat.setOutputPath(genJob, randomOuts); genJob.setOutputKeyClass(IntWritable.class); genJob.setOutputValueClass(IntWritable.class); genJob.setReducerClass(RandomGenReducer.class); genJob.setNumReduceTasks(1); genJob.waitForCompletion(true); printFiles(randomOuts, conf); // // Next, we read the big file in and regenerate the // original map. It's split into a number of parts. // (That number is 'intermediateReduces'.) // // We have many map tasks, each of which read at least one // of the output numbers. For each number read in, the // map task emits a key/value pair where the key is the // number and the value is "1". // // We have a single reduce task, which receives its input // sorted by the key emitted above. For each key, there will // be a certain number of "1" values. The reduce task sums // these values to compute how many times the given key was // emitted. // // The reduce task then emits a key/val pair where the key // is the number in question, and the value is the number of // times the key was emitted. This is the same format as the // original answer key (except that numbers emitted zero times // will not appear in the regenerated key.) The answer set // is split into a number of pieces. A final MapReduce job // will merge them. // // There's not really a need to go to 10 reduces here // instead of 1. But we want to test what happens when // you have multiple reduces at once. // int intermediateReduces = 10; Path intermediateOuts = new Path(testdir, "intermediateouts"); fs.delete(intermediateOuts, true); Job checkJob = Job.getInstance(conf); FileInputFormat.setInputPaths(checkJob, randomOuts); checkJob.setMapperClass(RandomCheckMapper.class); FileOutputFormat.setOutputPath(checkJob, intermediateOuts); checkJob.setOutputKeyClass(IntWritable.class); checkJob.setOutputValueClass(IntWritable.class); checkJob.setOutputFormatClass(MapFileOutputFormat.class); checkJob.setReducerClass(RandomCheckReducer.class); checkJob.setNumReduceTasks(intermediateReduces); checkJob.waitForCompletion(true); printFiles(intermediateOuts, conf); // // OK, now we take the output from the last job and // merge it down to a single file. The map() and reduce() // functions don't really do anything except reemit tuples. // But by having a single reduce task here, we end up merging // all the files. // Path finalOuts = new Path(testdir, "finalouts"); fs.delete(finalOuts, true); Job mergeJob = Job.getInstance(conf); FileInputFormat.setInputPaths(mergeJob, intermediateOuts); mergeJob.setInputFormatClass(SequenceFileInputFormat.class); mergeJob.setMapperClass(MergeMapper.class); FileOutputFormat.setOutputPath(mergeJob, finalOuts); mergeJob.setOutputKeyClass(IntWritable.class); mergeJob.setOutputValueClass(IntWritable.class); mergeJob.setOutputFormatClass(SequenceFileOutputFormat.class); mergeJob.setReducerClass(MergeReducer.class); mergeJob.setNumReduceTasks(1); mergeJob.waitForCompletion(true); printFiles(finalOuts, conf); // // Finally, we compare the reconstructed answer key with the // original one. Remember, we need to ignore zero-count items // in the original key. // boolean success = true; Path recomputedkey = new Path(finalOuts, "part-r-00000"); SequenceFile.Reader in = new SequenceFile.Reader(fs, recomputedkey, conf); int totalseen = 0; try { IntWritable key = new IntWritable(); IntWritable val = new IntWritable(); for (int i = 0; i < range; i++) { if (dist[i] == 0) { continue; } if (!in.next(key, val)) { System.err.println("Cannot read entry " + i); success = false; break; } else { if (!((key.get() == i) && (val.get() == dist[i]))) { System.err.println("Mismatch! Pos=" + key.get() + ", i=" + i + ", val=" + val.get() + ", dist[i]=" + dist[i]); success = false; } totalseen += val.get(); } } if (success) { if (in.next(key, val)) { System.err.println("Unnecessary lines in recomputed key!"); success = false; } } } finally { in.close(); } int originalTotal = 0; for (int i = 0; i < dist.length; i++) { originalTotal += dist[i]; } System.out.println("Original sum: " + originalTotal); System.out.println("Recomputed sum: " + totalseen); // // Write to "results" whether the test succeeded or not. // Path resultFile = new Path(testdir, "results"); BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(fs.create(resultFile))); try { bw.write("Success=" + success + "\n"); System.out.println("Success=" + success); } finally { bw.close(); } assertTrue("testMapRed failed", success); fs.delete(testdir, true);
5369 -1009652357apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None verify that 'format' really blew away all pre-existing files SATD_ADDED testCheckpoint() public void testCheckpoint() throws IOException Path file1 = new Path("checkpoint.dat"); Path file2 = new Path("checkpoint2.dat"); Collection namedirs = null; Configuration conf = new HdfsConfiguration(); conf.set(DFSConfigKeys.DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY, "0.0.0.0:0"); replication = (short) conf.getInt(DFSConfigKeys.DFS_REPLICATION_KEY, 3); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDatanodes).build(); cluster.waitActive(); FileSystem fileSys = cluster.getFileSystem(); try { // // verify that 'format' really blew away all pre-existing files // assertTrue(!fileSys.exists(file1)); assertTrue(!fileSys.exists(file2)); namedirs = cluster.getNameDirs(0); // // Create file1 // writeFile(fileSys, file1, replication); checkFile(fileSys, file1, replication); // // Take a checkpoint // SecondaryNameNode secondary = startSecondaryNameNode(conf); ErrorSimulator.initializeErrorSimulationEvent(4); secondary.doCheckpoint(); secondary.shutdown(); } finally { fileSys.close(); cluster.shutdown(); } // // Restart cluster and verify that file1 still exist. // cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDatanodes).format(false).build(); cluster.waitActive(); fileSys = cluster.getFileSystem(); Path tmpDir = new Path("/tmp_tmp"); try { // check that file1 still exists checkFile(fileSys, file1, replication); cleanupFile(fileSys, file1); // create new file file2 writeFile(fileSys, file2, replication); checkFile(fileSys, file2, replication); // // Take a checkpoint // SecondaryNameNode secondary = startSecondaryNameNode(conf); secondary.doCheckpoint(); fileSys.delete(tmpDir, true); fileSys.mkdirs(tmpDir); secondary.doCheckpoint(); secondary.shutdown(); } finally { fileSys.close(); cluster.shutdown(); } // // Restart cluster and verify that file2 exists and // file1 does not exist. // cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDatanodes).format(false).build(); cluster.waitActive(); fileSys = cluster.getFileSystem(); assertTrue(!fileSys.exists(file1)); assertTrue(fileSys.exists(tmpDir)); try { // verify that file2 exists checkFile(fileSys, file2, replication); } finally { fileSys.close(); cluster.shutdown(); } // file2 is left behind. testNameNodeImageSendFail(conf); testSecondaryNamenodeError1(conf); testSecondaryNamenodeError2(conf); testSecondaryNamenodeError3(conf); testNamedirError(conf, namedirs); testSecondaryFailsToReturnImage(conf); testStartup(conf);
5368 -1009652358apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None We should probably test for more of the file properties. SATD_ADDED checkFile(FileSystem, Path, int) static void checkFile(FileSystem fileSys, Path name, int repl) throws IOException assertTrue(fileSys.exists(name)); int replication = fileSys.getFileStatus(name).getReplication(); assertEquals("replication for " + name, repl, replication); // We should probably test for more of the file properties.
5367 -1009652359apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None XXX:Not to be used directly. SATD_ADDED getTimer() return timer; TimerTask getTimer()
5366 -1009652360apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None This file has three whole blocks. If the maxsplit size is half the block size, then there should be six splits. SATD_ADDED Text> createRecordReader(InputSplit, TaskAttemptContext) public RecordReader createRecordReader(InputSplit split, TaskAttemptContext context) throws IOException return null;
5365 -1009652361apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None maximum split size is 4 blocks SATD_ADDED Text> createRecordReader(InputSplit, TaskAttemptContext) public RecordReader createRecordReader(InputSplit split, TaskAttemptContext context) throws IOException return null;
5364 -1009652362apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None maximum split size is 3 blocks SATD_ADDED Text> createRecordReader(InputSplit, TaskAttemptContext) public RecordReader createRecordReader(InputSplit split, TaskAttemptContext context) throws IOException return null;
5363 -1009652363apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None maximum split size is 2 blocks SATD_ADDED Text> createRecordReader(InputSplit, TaskAttemptContext) public RecordReader createRecordReader(InputSplit split, TaskAttemptContext context) throws IOException return null;
5362 -1009652364apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Fix the filesystem by moving corrupted files to lost+found SATD_ADDED testFsckMove() public void testFsckMove() throws Exception DFSTestUtil util = new DFSTestUtil("TestFsck", 5, 3, 8 * 1024); MiniDFSCluster cluster = null; FileSystem fs = null; try { Configuration conf = new HdfsConfiguration(); conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 10000L); conf.setInt(DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_INTERVAL_KEY, 1); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(4).build(); String topDir = "/srcdat"; fs = cluster.getFileSystem(); cluster.waitActive(); util.createFiles(fs, topDir); util.waitReplication(fs, topDir, (short) 3); String outStr = runFsck(conf, 0, true, "/"); assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS)); // Corrupt a block by deleting it String[] fileNames = util.getFileNames(topDir); DFSClient dfsClient = new DFSClient(new InetSocketAddress("localhost", cluster.getNameNodePort()), conf); ExtendedBlock block = dfsClient.getNamenode().getBlockLocations(fileNames[0], 0, Long.MAX_VALUE).get(0).getBlock(); for (int i = 0; i < 4; i++) { File blockFile = MiniDFSCluster.getBlockFile(i, block); if (blockFile != null && blockFile.exists()) { assertTrue(blockFile.delete()); } } // We excpect the filesystem to be corrupted outStr = runFsck(conf, 1, false, "/"); while (!outStr.contains(NamenodeFsck.CORRUPT_STATUS)) { try { Thread.sleep(100); } catch (InterruptedException ignore) { } outStr = runFsck(conf, 1, false, "/"); } // Fix the filesystem by moving corrupted files to lost+found outStr = runFsck(conf, 1, true, "/", "-move"); assertTrue(outStr.contains(NamenodeFsck.CORRUPT_STATUS)); // Check to make sure we have healthy filesystem outStr = runFsck(conf, 0, true, "/"); assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS)); util.cleanup(fs, topDir); if (fs != null) { try { fs.close(); } catch (Exception e) { } } cluster.shutdown(); } finally { if (fs != null) { try { fs.close(); } catch (Exception e) { } } if (cluster != null) { cluster.shutdown(); } }
5361 -1009652365apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None TODO: test create schema TODO: test writable variants of data types SATD_ADDED testVerticaOutput() public void testVerticaOutput() throws Exception if (!AllTests.isSetup()) { return; } // TODO: test create schema // TODO: test writable variants of data types VerticaOutputFormat output = new VerticaOutputFormat(); Job job = getVerticaJob(); VerticaOutputFormat.setOutput(job, "mrtarget", true, "a int", "b boolean", "c char(1)", "d date", "f float", "t timestamp", "v varchar", "z varbinary"); output.checkOutputSpecs(job, true); TaskAttemptContext context = new TaskAttemptContextImpl(job.getConfiguration(), new TaskAttemptID()); VerticaRecordWriter writer = (VerticaRecordWriter) output.getRecordWriter(context); Text table = new Text(); table.set("mrtarget"); VerticaRecord record = VerticaOutputFormat.getValue(job.getConfiguration()); record.set(0, 125, true); record.set(1, true, true); record.set(2, 'c', true); record.set(3, Calendar.getInstance().getTime(), true); record.set(4, 234.526, true); record.set(5, Calendar.getInstance().getTime(), true); record.set(6, "foobar string", true); record.set(7, new byte[10], true); writer.write(table, record); writer.close(null);
5360 -1009652366apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None TODO: test values as hashmap of column names SATD_ADDED recordTest(List, List, DataOutputBuffer, DataInputBuffer, boolean) private String recordTest(List types, List values, DataOutputBuffer out, DataInputBuffer in, boolean date_string) throws IOException VerticaRecord record = new VerticaRecord(null, types, values, date_string); // TODO: test values as hashmap of column names // write values into an output buffer record.write(out); // copy to an input buffer in.reset(out.getData(), out.getLength()); // create a new record with new values List new_values = new ArrayList(); record = new VerticaRecord(null, types, new_values, date_string); // read back into values record.readFields(in); // compare values for (int i = 0; i < values.size(); i++) if (values.get(i) == null) assertSame("Vertica Record serialized value " + i + " is null", values.get(i), new_values.get(i)); else if (values.get(i).getClass().isArray()) { Object a = values.get(i); Object b = new_values.get(i); for (int j = 0; j < Array.getLength(a); j++) assertEquals("Vertica Record serialized value " + i + "[" + j + "] does not match", Array.get(a, j), Array.get(b, j)); } else { assertEquals("Vertica Record serialized value " + i + " does not match", values.get(i), new_values.get(i)); } // data in sql form return record.toSQLString();
5359 -1009652367apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Reverse: Move /nqdir0/qdir1/qdir20/nqdir30 to /nqdir0/qdir1/qdir21/ SATD_ADDED testSpaceCommands() public void testSpaceCommands() throws Exception final Configuration conf = new HdfsConfiguration(); // set a smaller block size so that we can test with smaller // diskspace quotas conf.set(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, "512"); conf.setBoolean("dfs.support.append", true); final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); final FileSystem fs = cluster.getFileSystem(); assertTrue("Not a HDFS: " + fs.getUri(), fs instanceof DistributedFileSystem); final DistributedFileSystem dfs = (DistributedFileSystem) fs; try { int fileLen = 1024; short replication = 3; int fileSpace = fileLen * replication; // create directory /nqdir0/qdir1/qdir20/nqdir30 assertTrue(dfs.mkdirs(new Path("/nqdir0/qdir1/qdir20/nqdir30"))); // set the quota of /nqdir0/qdir1 to 4 * fileSpace final Path quotaDir1 = new Path("/nqdir0/qdir1"); dfs.setQuota(quotaDir1, FSConstants.QUOTA_DONT_SET, 4 * fileSpace); ContentSummary c = dfs.getContentSummary(quotaDir1); assertEquals(c.getSpaceQuota(), 4 * fileSpace); // set the quota of /nqdir0/qdir1/qdir20 to 6 * fileSpace final Path quotaDir20 = new Path("/nqdir0/qdir1/qdir20"); dfs.setQuota(quotaDir20, FSConstants.QUOTA_DONT_SET, 6 * fileSpace); c = dfs.getContentSummary(quotaDir20); assertEquals(c.getSpaceQuota(), 6 * fileSpace); // Create /nqdir0/qdir1/qdir21 and set its space quota to 2 * fileSpace final Path quotaDir21 = new Path("/nqdir0/qdir1/qdir21"); assertTrue(dfs.mkdirs(quotaDir21)); dfs.setQuota(quotaDir21, FSConstants.QUOTA_DONT_SET, 2 * fileSpace); c = dfs.getContentSummary(quotaDir21); assertEquals(c.getSpaceQuota(), 2 * fileSpace); // 5: Create directory /nqdir0/qdir1/qdir21/nqdir32 Path tempPath = new Path(quotaDir21, "nqdir32"); assertTrue(dfs.mkdirs(tempPath)); // create a file under nqdir32/fileDir DFSTestUtil.createFile(dfs, new Path(tempPath, "fileDir/file1"), fileLen, replication, 0); c = dfs.getContentSummary(quotaDir21); assertEquals(c.getSpaceConsumed(), fileSpace); // Create a larger file /nqdir0/qdir1/qdir21/nqdir33/ boolean hasException = false; try { DFSTestUtil.createFile(dfs, new Path(quotaDir21, "nqdir33/file2"), 2 * fileLen, replication, 0); } catch (DSQuotaExceededException e) { hasException = true; } assertTrue(hasException); // delete nqdir33 assertTrue(dfs.delete(new Path(quotaDir21, "nqdir33"), true)); c = dfs.getContentSummary(quotaDir21); assertEquals(c.getSpaceConsumed(), fileSpace); assertEquals(c.getSpaceQuota(), 2 * fileSpace); // Verify space before the move: c = dfs.getContentSummary(quotaDir20); assertEquals(c.getSpaceConsumed(), 0); // Move /nqdir0/qdir1/qdir21/nqdir32 /nqdir0/qdir1/qdir20/nqdir30 Path dstPath = new Path(quotaDir20, "nqdir30"); Path srcPath = new Path(quotaDir21, "nqdir32"); assertTrue(dfs.rename(srcPath, dstPath)); // verify space after the move c = dfs.getContentSummary(quotaDir20); assertEquals(c.getSpaceConsumed(), fileSpace); // verify space for its parent c = dfs.getContentSummary(quotaDir1); assertEquals(c.getSpaceConsumed(), fileSpace); // verify space for source for the move c = dfs.getContentSummary(quotaDir21); assertEquals(c.getSpaceConsumed(), 0); final Path file2 = new Path(dstPath, "fileDir/file2"); int file2Len = 2 * fileLen; // create a larger file under /nqdir0/qdir1/qdir20/nqdir30 DFSTestUtil.createFile(dfs, file2, file2Len, replication, 0); c = dfs.getContentSummary(quotaDir20); assertEquals(c.getSpaceConsumed(), 3 * fileSpace); c = dfs.getContentSummary(quotaDir21); assertEquals(c.getSpaceConsumed(), 0); // Reverse: Move /nqdir0/qdir1/qdir20/nqdir30 to /nqdir0/qdir1/qdir21/ hasException = false; try { assertFalse(dfs.rename(dstPath, srcPath)); } catch (DSQuotaExceededException e) { hasException = true; } assertTrue(hasException); // make sure no intermediate directories left by failed rename assertFalse(dfs.exists(srcPath)); // directory should exist assertTrue(dfs.exists(dstPath)); // verify space after the failed move c = dfs.getContentSummary(quotaDir20); assertEquals(c.getSpaceConsumed(), 3 * fileSpace); c = dfs.getContentSummary(quotaDir21); assertEquals(c.getSpaceConsumed(), 0); // Test Append : // verify space quota c = dfs.getContentSummary(quotaDir1); assertEquals(c.getSpaceQuota(), 4 * fileSpace); // verify space before append; c = dfs.getContentSummary(dstPath); assertEquals(c.getSpaceConsumed(), 3 * fileSpace); OutputStream out = dfs.append(file2); // appending 1 fileLen should succeed out.write(new byte[fileLen]); out.close(); // after append file2Len += fileLen; // verify space after append; c = dfs.getContentSummary(dstPath); assertEquals(c.getSpaceConsumed(), 4 * fileSpace); // now increase the quota for quotaDir1 dfs.setQuota(quotaDir1, FSConstants.QUOTA_DONT_SET, 5 * fileSpace); // Now, appending more than 1 fileLen should result in an error out = dfs.append(file2); hasException = false; try { out.write(new byte[fileLen + 1024]); out.flush(); out.close(); } catch (DSQuotaExceededException e) { hasException = true; IOUtils.closeStream(out); } assertTrue(hasException); // after partial append file2Len += fileLen; // verify space after partial append c = dfs.getContentSummary(dstPath); assertEquals(c.getSpaceConsumed(), 5 * fileSpace); // Test set replication : // first reduce the replication dfs.setReplication(file2, (short) (replication - 1)); // verify that space is reduced by file2Len c = dfs.getContentSummary(dstPath); assertEquals(c.getSpaceConsumed(), 5 * fileSpace - file2Len); // now try to increase the replication and and expect an error. hasException = false; try { dfs.setReplication(file2, (short) (replication + 1)); } catch (DSQuotaExceededException e) { hasException = true; } assertTrue(hasException); // verify space consumed remains unchanged. c = dfs.getContentSummary(dstPath); assertEquals(c.getSpaceConsumed(), 5 * fileSpace - file2Len); // now increase the quota for quotaDir1 and quotaDir20 dfs.setQuota(quotaDir1, FSConstants.QUOTA_DONT_SET, 10 * fileSpace); dfs.setQuota(quotaDir20, FSConstants.QUOTA_DONT_SET, 10 * fileSpace); // then increasing replication should be ok. dfs.setReplication(file2, (short) (replication + 1)); // verify increase in space c = dfs.getContentSummary(dstPath); assertEquals(c.getSpaceConsumed(), 5 * fileSpace + file2Len); } finally { cluster.shutdown(); }
5358 -1009652368apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Move /nqdir0/qdir1/qdir21/nqdir32 /nqdir0/qdir1/qdir20/nqdir30 SATD_ADDED testSpaceCommands() public void testSpaceCommands() throws Exception final Configuration conf = new HdfsConfiguration(); // set a smaller block size so that we can test with smaller // diskspace quotas conf.set(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, "512"); conf.setBoolean("dfs.support.append", true); final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); final FileSystem fs = cluster.getFileSystem(); assertTrue("Not a HDFS: " + fs.getUri(), fs instanceof DistributedFileSystem); final DistributedFileSystem dfs = (DistributedFileSystem) fs; try { int fileLen = 1024; short replication = 3; int fileSpace = fileLen * replication; // create directory /nqdir0/qdir1/qdir20/nqdir30 assertTrue(dfs.mkdirs(new Path("/nqdir0/qdir1/qdir20/nqdir30"))); // set the quota of /nqdir0/qdir1 to 4 * fileSpace final Path quotaDir1 = new Path("/nqdir0/qdir1"); dfs.setQuota(quotaDir1, FSConstants.QUOTA_DONT_SET, 4 * fileSpace); ContentSummary c = dfs.getContentSummary(quotaDir1); assertEquals(c.getSpaceQuota(), 4 * fileSpace); // set the quota of /nqdir0/qdir1/qdir20 to 6 * fileSpace final Path quotaDir20 = new Path("/nqdir0/qdir1/qdir20"); dfs.setQuota(quotaDir20, FSConstants.QUOTA_DONT_SET, 6 * fileSpace); c = dfs.getContentSummary(quotaDir20); assertEquals(c.getSpaceQuota(), 6 * fileSpace); // Create /nqdir0/qdir1/qdir21 and set its space quota to 2 * fileSpace final Path quotaDir21 = new Path("/nqdir0/qdir1/qdir21"); assertTrue(dfs.mkdirs(quotaDir21)); dfs.setQuota(quotaDir21, FSConstants.QUOTA_DONT_SET, 2 * fileSpace); c = dfs.getContentSummary(quotaDir21); assertEquals(c.getSpaceQuota(), 2 * fileSpace); // 5: Create directory /nqdir0/qdir1/qdir21/nqdir32 Path tempPath = new Path(quotaDir21, "nqdir32"); assertTrue(dfs.mkdirs(tempPath)); // create a file under nqdir32/fileDir DFSTestUtil.createFile(dfs, new Path(tempPath, "fileDir/file1"), fileLen, replication, 0); c = dfs.getContentSummary(quotaDir21); assertEquals(c.getSpaceConsumed(), fileSpace); // Create a larger file /nqdir0/qdir1/qdir21/nqdir33/ boolean hasException = false; try { DFSTestUtil.createFile(dfs, new Path(quotaDir21, "nqdir33/file2"), 2 * fileLen, replication, 0); } catch (DSQuotaExceededException e) { hasException = true; } assertTrue(hasException); // delete nqdir33 assertTrue(dfs.delete(new Path(quotaDir21, "nqdir33"), true)); c = dfs.getContentSummary(quotaDir21); assertEquals(c.getSpaceConsumed(), fileSpace); assertEquals(c.getSpaceQuota(), 2 * fileSpace); // Verify space before the move: c = dfs.getContentSummary(quotaDir20); assertEquals(c.getSpaceConsumed(), 0); // Move /nqdir0/qdir1/qdir21/nqdir32 /nqdir0/qdir1/qdir20/nqdir30 Path dstPath = new Path(quotaDir20, "nqdir30"); Path srcPath = new Path(quotaDir21, "nqdir32"); assertTrue(dfs.rename(srcPath, dstPath)); // verify space after the move c = dfs.getContentSummary(quotaDir20); assertEquals(c.getSpaceConsumed(), fileSpace); // verify space for its parent c = dfs.getContentSummary(quotaDir1); assertEquals(c.getSpaceConsumed(), fileSpace); // verify space for source for the move c = dfs.getContentSummary(quotaDir21); assertEquals(c.getSpaceConsumed(), 0); final Path file2 = new Path(dstPath, "fileDir/file2"); int file2Len = 2 * fileLen; // create a larger file under /nqdir0/qdir1/qdir20/nqdir30 DFSTestUtil.createFile(dfs, file2, file2Len, replication, 0); c = dfs.getContentSummary(quotaDir20); assertEquals(c.getSpaceConsumed(), 3 * fileSpace); c = dfs.getContentSummary(quotaDir21); assertEquals(c.getSpaceConsumed(), 0); // Reverse: Move /nqdir0/qdir1/qdir20/nqdir30 to /nqdir0/qdir1/qdir21/ hasException = false; try { assertFalse(dfs.rename(dstPath, srcPath)); } catch (DSQuotaExceededException e) { hasException = true; } assertTrue(hasException); // make sure no intermediate directories left by failed rename assertFalse(dfs.exists(srcPath)); // directory should exist assertTrue(dfs.exists(dstPath)); // verify space after the failed move c = dfs.getContentSummary(quotaDir20); assertEquals(c.getSpaceConsumed(), 3 * fileSpace); c = dfs.getContentSummary(quotaDir21); assertEquals(c.getSpaceConsumed(), 0); // Test Append : // verify space quota c = dfs.getContentSummary(quotaDir1); assertEquals(c.getSpaceQuota(), 4 * fileSpace); // verify space before append; c = dfs.getContentSummary(dstPath); assertEquals(c.getSpaceConsumed(), 3 * fileSpace); OutputStream out = dfs.append(file2); // appending 1 fileLen should succeed out.write(new byte[fileLen]); out.close(); // after append file2Len += fileLen; // verify space after append; c = dfs.getContentSummary(dstPath); assertEquals(c.getSpaceConsumed(), 4 * fileSpace); // now increase the quota for quotaDir1 dfs.setQuota(quotaDir1, FSConstants.QUOTA_DONT_SET, 5 * fileSpace); // Now, appending more than 1 fileLen should result in an error out = dfs.append(file2); hasException = false; try { out.write(new byte[fileLen + 1024]); out.flush(); out.close(); } catch (DSQuotaExceededException e) { hasException = true; IOUtils.closeStream(out); } assertTrue(hasException); // after partial append file2Len += fileLen; // verify space after partial append c = dfs.getContentSummary(dstPath); assertEquals(c.getSpaceConsumed(), 5 * fileSpace); // Test set replication : // first reduce the replication dfs.setReplication(file2, (short) (replication - 1)); // verify that space is reduced by file2Len c = dfs.getContentSummary(dstPath); assertEquals(c.getSpaceConsumed(), 5 * fileSpace - file2Len); // now try to increase the replication and and expect an error. hasException = false; try { dfs.setReplication(file2, (short) (replication + 1)); } catch (DSQuotaExceededException e) { hasException = true; } assertTrue(hasException); // verify space consumed remains unchanged. c = dfs.getContentSummary(dstPath); assertEquals(c.getSpaceConsumed(), 5 * fileSpace - file2Len); // now increase the quota for quotaDir1 and quotaDir20 dfs.setQuota(quotaDir1, FSConstants.QUOTA_DONT_SET, 10 * fileSpace); dfs.setQuota(quotaDir20, FSConstants.QUOTA_DONT_SET, 10 * fileSpace); // then increasing replication should be ok. dfs.setReplication(file2, (short) (replication + 1)); // verify increase in space c = dfs.getContentSummary(dstPath); assertEquals(c.getSpaceConsumed(), 5 * fileSpace + file2Len); } finally { cluster.shutdown(); }
5357 -1009652369apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None 16: Move /nqdir0/qdir30 /nqdir0/qdir1/qdir20 SATD_ADDED testNamespaceCommands() public void testNamespaceCommands() throws Exception final Configuration conf = new HdfsConfiguration(); final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); final FileSystem fs = cluster.getFileSystem(); assertTrue("Not a HDFS: " + fs.getUri(), fs instanceof DistributedFileSystem); final DistributedFileSystem dfs = (DistributedFileSystem) fs; try { // 1: create directory /nqdir0/qdir1/qdir20/nqdir30 assertTrue(dfs.mkdirs(new Path("/nqdir0/qdir1/qdir20/nqdir30"))); // 2: set the quota of /nqdir0/qdir1 to be 6 final Path quotaDir1 = new Path("/nqdir0/qdir1"); dfs.setQuota(quotaDir1, 6, FSConstants.QUOTA_DONT_SET); ContentSummary c = dfs.getContentSummary(quotaDir1); assertEquals(c.getDirectoryCount(), 3); assertEquals(c.getQuota(), 6); // 3: set the quota of /nqdir0/qdir1/qdir20 to be 7 final Path quotaDir2 = new Path("/nqdir0/qdir1/qdir20"); dfs.setQuota(quotaDir2, 7, FSConstants.QUOTA_DONT_SET); c = dfs.getContentSummary(quotaDir2); assertEquals(c.getDirectoryCount(), 2); assertEquals(c.getQuota(), 7); // 4: Create directory /nqdir0/qdir1/qdir21 and set its quota to 2 final Path quotaDir3 = new Path("/nqdir0/qdir1/qdir21"); assertTrue(dfs.mkdirs(quotaDir3)); dfs.setQuota(quotaDir3, 2, FSConstants.QUOTA_DONT_SET); c = dfs.getContentSummary(quotaDir3); assertEquals(c.getDirectoryCount(), 1); assertEquals(c.getQuota(), 2); // 5: Create directory /nqdir0/qdir1/qdir21/nqdir32 Path tempPath = new Path(quotaDir3, "nqdir32"); assertTrue(dfs.mkdirs(tempPath)); c = dfs.getContentSummary(quotaDir3); assertEquals(c.getDirectoryCount(), 2); assertEquals(c.getQuota(), 2); // 6: Create directory /nqdir0/qdir1/qdir21/nqdir33 tempPath = new Path(quotaDir3, "nqdir33"); boolean hasException = false; try { assertFalse(dfs.mkdirs(tempPath)); } catch (NSQuotaExceededException e) { hasException = true; } assertTrue(hasException); c = dfs.getContentSummary(quotaDir3); assertEquals(c.getDirectoryCount(), 2); assertEquals(c.getQuota(), 2); // 7: Create directory /nqdir0/qdir1/qdir20/nqdir31 tempPath = new Path(quotaDir2, "nqdir31"); assertTrue(dfs.mkdirs(tempPath)); c = dfs.getContentSummary(quotaDir2); assertEquals(c.getDirectoryCount(), 3); assertEquals(c.getQuota(), 7); c = dfs.getContentSummary(quotaDir1); assertEquals(c.getDirectoryCount(), 6); assertEquals(c.getQuota(), 6); // 8: Create directory /nqdir0/qdir1/qdir20/nqdir33 tempPath = new Path(quotaDir2, "nqdir33"); hasException = false; try { assertFalse(dfs.mkdirs(tempPath)); } catch (NSQuotaExceededException e) { hasException = true; } assertTrue(hasException); // 9: Move /nqdir0/qdir1/qdir21/nqdir32 /nqdir0/qdir1/qdir20/nqdir30 tempPath = new Path(quotaDir2, "nqdir30"); dfs.rename(new Path(quotaDir3, "nqdir32"), tempPath); c = dfs.getContentSummary(quotaDir2); assertEquals(c.getDirectoryCount(), 4); assertEquals(c.getQuota(), 7); c = dfs.getContentSummary(quotaDir1); assertEquals(c.getDirectoryCount(), 6); assertEquals(c.getQuota(), 6); // 10: Move /nqdir0/qdir1/qdir20/nqdir30 to /nqdir0/qdir1/qdir21 hasException = false; try { assertFalse(dfs.rename(tempPath, quotaDir3)); } catch (NSQuotaExceededException e) { hasException = true; } assertTrue(hasException); assertTrue(dfs.exists(tempPath)); assertFalse(dfs.exists(new Path(quotaDir3, "nqdir30"))); // 10.a: Rename /nqdir0/qdir1/qdir20/nqdir30 to /nqdir0/qdir1/qdir21/nqdir32 hasException = false; try { assertFalse(dfs.rename(tempPath, new Path(quotaDir3, "nqdir32"))); } catch (QuotaExceededException e) { hasException = true; } assertTrue(hasException); assertTrue(dfs.exists(tempPath)); assertFalse(dfs.exists(new Path(quotaDir3, "nqdir32"))); // 11: Move /nqdir0/qdir1/qdir20/nqdir30 to /nqdir0 assertTrue(dfs.rename(tempPath, new Path("/nqdir0"))); c = dfs.getContentSummary(quotaDir2); assertEquals(c.getDirectoryCount(), 2); assertEquals(c.getQuota(), 7); c = dfs.getContentSummary(quotaDir1); assertEquals(c.getDirectoryCount(), 4); assertEquals(c.getQuota(), 6); // 12: Create directory /nqdir0/nqdir30/nqdir33 assertTrue(dfs.mkdirs(new Path("/nqdir0/nqdir30/nqdir33"))); // 13: Move /nqdir0/nqdir30 /nqdir0/qdir1/qdir20/qdir30 hasException = false; try { assertFalse(dfs.rename(new Path("/nqdir0/nqdir30"), tempPath)); } catch (NSQuotaExceededException e) { hasException = true; } assertTrue(hasException); // 14: Move /nqdir0/qdir1/qdir21 /nqdir0/qdir1/qdir20 assertTrue(dfs.rename(quotaDir3, quotaDir2)); c = dfs.getContentSummary(quotaDir1); assertEquals(c.getDirectoryCount(), 4); assertEquals(c.getQuota(), 6); c = dfs.getContentSummary(quotaDir2); assertEquals(c.getDirectoryCount(), 3); assertEquals(c.getQuota(), 7); tempPath = new Path(quotaDir2, "qdir21"); c = dfs.getContentSummary(tempPath); assertEquals(c.getDirectoryCount(), 1); assertEquals(c.getQuota(), 2); // 15: Delete /nqdir0/qdir1/qdir20/qdir21 dfs.delete(tempPath, true); c = dfs.getContentSummary(quotaDir2); assertEquals(c.getDirectoryCount(), 2); assertEquals(c.getQuota(), 7); c = dfs.getContentSummary(quotaDir1); assertEquals(c.getDirectoryCount(), 3); assertEquals(c.getQuota(), 6); // 16: Move /nqdir0/qdir30 /nqdir0/qdir1/qdir20 assertTrue(dfs.rename(new Path("/nqdir0/nqdir30"), quotaDir2)); c = dfs.getContentSummary(quotaDir2); assertEquals(c.getDirectoryCount(), 5); assertEquals(c.getQuota(), 7); c = dfs.getContentSummary(quotaDir1); assertEquals(c.getDirectoryCount(), 6); assertEquals(c.getQuota(), 6); } finally { cluster.shutdown(); }
5356 -1009652370apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None 14: Move /nqdir0/qdir1/qdir21 /nqdir0/qdir1/qdir20 SATD_ADDED testNamespaceCommands() public void testNamespaceCommands() throws Exception final Configuration conf = new HdfsConfiguration(); final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); final FileSystem fs = cluster.getFileSystem(); assertTrue("Not a HDFS: " + fs.getUri(), fs instanceof DistributedFileSystem); final DistributedFileSystem dfs = (DistributedFileSystem) fs; try { // 1: create directory /nqdir0/qdir1/qdir20/nqdir30 assertTrue(dfs.mkdirs(new Path("/nqdir0/qdir1/qdir20/nqdir30"))); // 2: set the quota of /nqdir0/qdir1 to be 6 final Path quotaDir1 = new Path("/nqdir0/qdir1"); dfs.setQuota(quotaDir1, 6, FSConstants.QUOTA_DONT_SET); ContentSummary c = dfs.getContentSummary(quotaDir1); assertEquals(c.getDirectoryCount(), 3); assertEquals(c.getQuota(), 6); // 3: set the quota of /nqdir0/qdir1/qdir20 to be 7 final Path quotaDir2 = new Path("/nqdir0/qdir1/qdir20"); dfs.setQuota(quotaDir2, 7, FSConstants.QUOTA_DONT_SET); c = dfs.getContentSummary(quotaDir2); assertEquals(c.getDirectoryCount(), 2); assertEquals(c.getQuota(), 7); // 4: Create directory /nqdir0/qdir1/qdir21 and set its quota to 2 final Path quotaDir3 = new Path("/nqdir0/qdir1/qdir21"); assertTrue(dfs.mkdirs(quotaDir3)); dfs.setQuota(quotaDir3, 2, FSConstants.QUOTA_DONT_SET); c = dfs.getContentSummary(quotaDir3); assertEquals(c.getDirectoryCount(), 1); assertEquals(c.getQuota(), 2); // 5: Create directory /nqdir0/qdir1/qdir21/nqdir32 Path tempPath = new Path(quotaDir3, "nqdir32"); assertTrue(dfs.mkdirs(tempPath)); c = dfs.getContentSummary(quotaDir3); assertEquals(c.getDirectoryCount(), 2); assertEquals(c.getQuota(), 2); // 6: Create directory /nqdir0/qdir1/qdir21/nqdir33 tempPath = new Path(quotaDir3, "nqdir33"); boolean hasException = false; try { assertFalse(dfs.mkdirs(tempPath)); } catch (NSQuotaExceededException e) { hasException = true; } assertTrue(hasException); c = dfs.getContentSummary(quotaDir3); assertEquals(c.getDirectoryCount(), 2); assertEquals(c.getQuota(), 2); // 7: Create directory /nqdir0/qdir1/qdir20/nqdir31 tempPath = new Path(quotaDir2, "nqdir31"); assertTrue(dfs.mkdirs(tempPath)); c = dfs.getContentSummary(quotaDir2); assertEquals(c.getDirectoryCount(), 3); assertEquals(c.getQuota(), 7); c = dfs.getContentSummary(quotaDir1); assertEquals(c.getDirectoryCount(), 6); assertEquals(c.getQuota(), 6); // 8: Create directory /nqdir0/qdir1/qdir20/nqdir33 tempPath = new Path(quotaDir2, "nqdir33"); hasException = false; try { assertFalse(dfs.mkdirs(tempPath)); } catch (NSQuotaExceededException e) { hasException = true; } assertTrue(hasException); // 9: Move /nqdir0/qdir1/qdir21/nqdir32 /nqdir0/qdir1/qdir20/nqdir30 tempPath = new Path(quotaDir2, "nqdir30"); dfs.rename(new Path(quotaDir3, "nqdir32"), tempPath); c = dfs.getContentSummary(quotaDir2); assertEquals(c.getDirectoryCount(), 4); assertEquals(c.getQuota(), 7); c = dfs.getContentSummary(quotaDir1); assertEquals(c.getDirectoryCount(), 6); assertEquals(c.getQuota(), 6); // 10: Move /nqdir0/qdir1/qdir20/nqdir30 to /nqdir0/qdir1/qdir21 hasException = false; try { assertFalse(dfs.rename(tempPath, quotaDir3)); } catch (NSQuotaExceededException e) { hasException = true; } assertTrue(hasException); assertTrue(dfs.exists(tempPath)); assertFalse(dfs.exists(new Path(quotaDir3, "nqdir30"))); // 10.a: Rename /nqdir0/qdir1/qdir20/nqdir30 to /nqdir0/qdir1/qdir21/nqdir32 hasException = false; try { assertFalse(dfs.rename(tempPath, new Path(quotaDir3, "nqdir32"))); } catch (QuotaExceededException e) { hasException = true; } assertTrue(hasException); assertTrue(dfs.exists(tempPath)); assertFalse(dfs.exists(new Path(quotaDir3, "nqdir32"))); // 11: Move /nqdir0/qdir1/qdir20/nqdir30 to /nqdir0 assertTrue(dfs.rename(tempPath, new Path("/nqdir0"))); c = dfs.getContentSummary(quotaDir2); assertEquals(c.getDirectoryCount(), 2); assertEquals(c.getQuota(), 7); c = dfs.getContentSummary(quotaDir1); assertEquals(c.getDirectoryCount(), 4); assertEquals(c.getQuota(), 6); // 12: Create directory /nqdir0/nqdir30/nqdir33 assertTrue(dfs.mkdirs(new Path("/nqdir0/nqdir30/nqdir33"))); // 13: Move /nqdir0/nqdir30 /nqdir0/qdir1/qdir20/qdir30 hasException = false; try { assertFalse(dfs.rename(new Path("/nqdir0/nqdir30"), tempPath)); } catch (NSQuotaExceededException e) { hasException = true; } assertTrue(hasException); // 14: Move /nqdir0/qdir1/qdir21 /nqdir0/qdir1/qdir20 assertTrue(dfs.rename(quotaDir3, quotaDir2)); c = dfs.getContentSummary(quotaDir1); assertEquals(c.getDirectoryCount(), 4); assertEquals(c.getQuota(), 6); c = dfs.getContentSummary(quotaDir2); assertEquals(c.getDirectoryCount(), 3); assertEquals(c.getQuota(), 7); tempPath = new Path(quotaDir2, "qdir21"); c = dfs.getContentSummary(tempPath); assertEquals(c.getDirectoryCount(), 1); assertEquals(c.getQuota(), 2); // 15: Delete /nqdir0/qdir1/qdir20/qdir21 dfs.delete(tempPath, true); c = dfs.getContentSummary(quotaDir2); assertEquals(c.getDirectoryCount(), 2); assertEquals(c.getQuota(), 7); c = dfs.getContentSummary(quotaDir1); assertEquals(c.getDirectoryCount(), 3); assertEquals(c.getQuota(), 6); // 16: Move /nqdir0/qdir30 /nqdir0/qdir1/qdir20 assertTrue(dfs.rename(new Path("/nqdir0/nqdir30"), quotaDir2)); c = dfs.getContentSummary(quotaDir2); assertEquals(c.getDirectoryCount(), 5); assertEquals(c.getQuota(), 7); c = dfs.getContentSummary(quotaDir1); assertEquals(c.getDirectoryCount(), 6); assertEquals(c.getQuota(), 6); } finally { cluster.shutdown(); }
5355 -1009652371apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None 13: Move /nqdir0/nqdir30 /nqdir0/qdir1/qdir20/qdir30 SATD_ADDED testNamespaceCommands() public void testNamespaceCommands() throws Exception final Configuration conf = new HdfsConfiguration(); final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); final FileSystem fs = cluster.getFileSystem(); assertTrue("Not a HDFS: " + fs.getUri(), fs instanceof DistributedFileSystem); final DistributedFileSystem dfs = (DistributedFileSystem) fs; try { // 1: create directory /nqdir0/qdir1/qdir20/nqdir30 assertTrue(dfs.mkdirs(new Path("/nqdir0/qdir1/qdir20/nqdir30"))); // 2: set the quota of /nqdir0/qdir1 to be 6 final Path quotaDir1 = new Path("/nqdir0/qdir1"); dfs.setQuota(quotaDir1, 6, FSConstants.QUOTA_DONT_SET); ContentSummary c = dfs.getContentSummary(quotaDir1); assertEquals(c.getDirectoryCount(), 3); assertEquals(c.getQuota(), 6); // 3: set the quota of /nqdir0/qdir1/qdir20 to be 7 final Path quotaDir2 = new Path("/nqdir0/qdir1/qdir20"); dfs.setQuota(quotaDir2, 7, FSConstants.QUOTA_DONT_SET); c = dfs.getContentSummary(quotaDir2); assertEquals(c.getDirectoryCount(), 2); assertEquals(c.getQuota(), 7); // 4: Create directory /nqdir0/qdir1/qdir21 and set its quota to 2 final Path quotaDir3 = new Path("/nqdir0/qdir1/qdir21"); assertTrue(dfs.mkdirs(quotaDir3)); dfs.setQuota(quotaDir3, 2, FSConstants.QUOTA_DONT_SET); c = dfs.getContentSummary(quotaDir3); assertEquals(c.getDirectoryCount(), 1); assertEquals(c.getQuota(), 2); // 5: Create directory /nqdir0/qdir1/qdir21/nqdir32 Path tempPath = new Path(quotaDir3, "nqdir32"); assertTrue(dfs.mkdirs(tempPath)); c = dfs.getContentSummary(quotaDir3); assertEquals(c.getDirectoryCount(), 2); assertEquals(c.getQuota(), 2); // 6: Create directory /nqdir0/qdir1/qdir21/nqdir33 tempPath = new Path(quotaDir3, "nqdir33"); boolean hasException = false; try { assertFalse(dfs.mkdirs(tempPath)); } catch (NSQuotaExceededException e) { hasException = true; } assertTrue(hasException); c = dfs.getContentSummary(quotaDir3); assertEquals(c.getDirectoryCount(), 2); assertEquals(c.getQuota(), 2); // 7: Create directory /nqdir0/qdir1/qdir20/nqdir31 tempPath = new Path(quotaDir2, "nqdir31"); assertTrue(dfs.mkdirs(tempPath)); c = dfs.getContentSummary(quotaDir2); assertEquals(c.getDirectoryCount(), 3); assertEquals(c.getQuota(), 7); c = dfs.getContentSummary(quotaDir1); assertEquals(c.getDirectoryCount(), 6); assertEquals(c.getQuota(), 6); // 8: Create directory /nqdir0/qdir1/qdir20/nqdir33 tempPath = new Path(quotaDir2, "nqdir33"); hasException = false; try { assertFalse(dfs.mkdirs(tempPath)); } catch (NSQuotaExceededException e) { hasException = true; } assertTrue(hasException); // 9: Move /nqdir0/qdir1/qdir21/nqdir32 /nqdir0/qdir1/qdir20/nqdir30 tempPath = new Path(quotaDir2, "nqdir30"); dfs.rename(new Path(quotaDir3, "nqdir32"), tempPath); c = dfs.getContentSummary(quotaDir2); assertEquals(c.getDirectoryCount(), 4); assertEquals(c.getQuota(), 7); c = dfs.getContentSummary(quotaDir1); assertEquals(c.getDirectoryCount(), 6); assertEquals(c.getQuota(), 6); // 10: Move /nqdir0/qdir1/qdir20/nqdir30 to /nqdir0/qdir1/qdir21 hasException = false; try { assertFalse(dfs.rename(tempPath, quotaDir3)); } catch (NSQuotaExceededException e) { hasException = true; } assertTrue(hasException); assertTrue(dfs.exists(tempPath)); assertFalse(dfs.exists(new Path(quotaDir3, "nqdir30"))); // 10.a: Rename /nqdir0/qdir1/qdir20/nqdir30 to /nqdir0/qdir1/qdir21/nqdir32 hasException = false; try { assertFalse(dfs.rename(tempPath, new Path(quotaDir3, "nqdir32"))); } catch (QuotaExceededException e) { hasException = true; } assertTrue(hasException); assertTrue(dfs.exists(tempPath)); assertFalse(dfs.exists(new Path(quotaDir3, "nqdir32"))); // 11: Move /nqdir0/qdir1/qdir20/nqdir30 to /nqdir0 assertTrue(dfs.rename(tempPath, new Path("/nqdir0"))); c = dfs.getContentSummary(quotaDir2); assertEquals(c.getDirectoryCount(), 2); assertEquals(c.getQuota(), 7); c = dfs.getContentSummary(quotaDir1); assertEquals(c.getDirectoryCount(), 4); assertEquals(c.getQuota(), 6); // 12: Create directory /nqdir0/nqdir30/nqdir33 assertTrue(dfs.mkdirs(new Path("/nqdir0/nqdir30/nqdir33"))); // 13: Move /nqdir0/nqdir30 /nqdir0/qdir1/qdir20/qdir30 hasException = false; try { assertFalse(dfs.rename(new Path("/nqdir0/nqdir30"), tempPath)); } catch (NSQuotaExceededException e) { hasException = true; } assertTrue(hasException); // 14: Move /nqdir0/qdir1/qdir21 /nqdir0/qdir1/qdir20 assertTrue(dfs.rename(quotaDir3, quotaDir2)); c = dfs.getContentSummary(quotaDir1); assertEquals(c.getDirectoryCount(), 4); assertEquals(c.getQuota(), 6); c = dfs.getContentSummary(quotaDir2); assertEquals(c.getDirectoryCount(), 3); assertEquals(c.getQuota(), 7); tempPath = new Path(quotaDir2, "qdir21"); c = dfs.getContentSummary(tempPath); assertEquals(c.getDirectoryCount(), 1); assertEquals(c.getQuota(), 2); // 15: Delete /nqdir0/qdir1/qdir20/qdir21 dfs.delete(tempPath, true); c = dfs.getContentSummary(quotaDir2); assertEquals(c.getDirectoryCount(), 2); assertEquals(c.getQuota(), 7); c = dfs.getContentSummary(quotaDir1); assertEquals(c.getDirectoryCount(), 3); assertEquals(c.getQuota(), 6); // 16: Move /nqdir0/qdir30 /nqdir0/qdir1/qdir20 assertTrue(dfs.rename(new Path("/nqdir0/nqdir30"), quotaDir2)); c = dfs.getContentSummary(quotaDir2); assertEquals(c.getDirectoryCount(), 5); assertEquals(c.getQuota(), 7); c = dfs.getContentSummary(quotaDir1); assertEquals(c.getDirectoryCount(), 6); assertEquals(c.getQuota(), 6); } finally { cluster.shutdown(); }
5354 -1009652372apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None 11: Move /nqdir0/qdir1/qdir20/nqdir30 to /nqdir0 SATD_ADDED testNamespaceCommands() public void testNamespaceCommands() throws Exception final Configuration conf = new HdfsConfiguration(); final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); final FileSystem fs = cluster.getFileSystem(); assertTrue("Not a HDFS: " + fs.getUri(), fs instanceof DistributedFileSystem); final DistributedFileSystem dfs = (DistributedFileSystem) fs; try { // 1: create directory /nqdir0/qdir1/qdir20/nqdir30 assertTrue(dfs.mkdirs(new Path("/nqdir0/qdir1/qdir20/nqdir30"))); // 2: set the quota of /nqdir0/qdir1 to be 6 final Path quotaDir1 = new Path("/nqdir0/qdir1"); dfs.setQuota(quotaDir1, 6, FSConstants.QUOTA_DONT_SET); ContentSummary c = dfs.getContentSummary(quotaDir1); assertEquals(c.getDirectoryCount(), 3); assertEquals(c.getQuota(), 6); // 3: set the quota of /nqdir0/qdir1/qdir20 to be 7 final Path quotaDir2 = new Path("/nqdir0/qdir1/qdir20"); dfs.setQuota(quotaDir2, 7, FSConstants.QUOTA_DONT_SET); c = dfs.getContentSummary(quotaDir2); assertEquals(c.getDirectoryCount(), 2); assertEquals(c.getQuota(), 7); // 4: Create directory /nqdir0/qdir1/qdir21 and set its quota to 2 final Path quotaDir3 = new Path("/nqdir0/qdir1/qdir21"); assertTrue(dfs.mkdirs(quotaDir3)); dfs.setQuota(quotaDir3, 2, FSConstants.QUOTA_DONT_SET); c = dfs.getContentSummary(quotaDir3); assertEquals(c.getDirectoryCount(), 1); assertEquals(c.getQuota(), 2); // 5: Create directory /nqdir0/qdir1/qdir21/nqdir32 Path tempPath = new Path(quotaDir3, "nqdir32"); assertTrue(dfs.mkdirs(tempPath)); c = dfs.getContentSummary(quotaDir3); assertEquals(c.getDirectoryCount(), 2); assertEquals(c.getQuota(), 2); // 6: Create directory /nqdir0/qdir1/qdir21/nqdir33 tempPath = new Path(quotaDir3, "nqdir33"); boolean hasException = false; try { assertFalse(dfs.mkdirs(tempPath)); } catch (NSQuotaExceededException e) { hasException = true; } assertTrue(hasException); c = dfs.getContentSummary(quotaDir3); assertEquals(c.getDirectoryCount(), 2); assertEquals(c.getQuota(), 2); // 7: Create directory /nqdir0/qdir1/qdir20/nqdir31 tempPath = new Path(quotaDir2, "nqdir31"); assertTrue(dfs.mkdirs(tempPath)); c = dfs.getContentSummary(quotaDir2); assertEquals(c.getDirectoryCount(), 3); assertEquals(c.getQuota(), 7); c = dfs.getContentSummary(quotaDir1); assertEquals(c.getDirectoryCount(), 6); assertEquals(c.getQuota(), 6); // 8: Create directory /nqdir0/qdir1/qdir20/nqdir33 tempPath = new Path(quotaDir2, "nqdir33"); hasException = false; try { assertFalse(dfs.mkdirs(tempPath)); } catch (NSQuotaExceededException e) { hasException = true; } assertTrue(hasException); // 9: Move /nqdir0/qdir1/qdir21/nqdir32 /nqdir0/qdir1/qdir20/nqdir30 tempPath = new Path(quotaDir2, "nqdir30"); dfs.rename(new Path(quotaDir3, "nqdir32"), tempPath); c = dfs.getContentSummary(quotaDir2); assertEquals(c.getDirectoryCount(), 4); assertEquals(c.getQuota(), 7); c = dfs.getContentSummary(quotaDir1); assertEquals(c.getDirectoryCount(), 6); assertEquals(c.getQuota(), 6); // 10: Move /nqdir0/qdir1/qdir20/nqdir30 to /nqdir0/qdir1/qdir21 hasException = false; try { assertFalse(dfs.rename(tempPath, quotaDir3)); } catch (NSQuotaExceededException e) { hasException = true; } assertTrue(hasException); assertTrue(dfs.exists(tempPath)); assertFalse(dfs.exists(new Path(quotaDir3, "nqdir30"))); // 10.a: Rename /nqdir0/qdir1/qdir20/nqdir30 to /nqdir0/qdir1/qdir21/nqdir32 hasException = false; try { assertFalse(dfs.rename(tempPath, new Path(quotaDir3, "nqdir32"))); } catch (QuotaExceededException e) { hasException = true; } assertTrue(hasException); assertTrue(dfs.exists(tempPath)); assertFalse(dfs.exists(new Path(quotaDir3, "nqdir32"))); // 11: Move /nqdir0/qdir1/qdir20/nqdir30 to /nqdir0 assertTrue(dfs.rename(tempPath, new Path("/nqdir0"))); c = dfs.getContentSummary(quotaDir2); assertEquals(c.getDirectoryCount(), 2); assertEquals(c.getQuota(), 7); c = dfs.getContentSummary(quotaDir1); assertEquals(c.getDirectoryCount(), 4); assertEquals(c.getQuota(), 6); // 12: Create directory /nqdir0/nqdir30/nqdir33 assertTrue(dfs.mkdirs(new Path("/nqdir0/nqdir30/nqdir33"))); // 13: Move /nqdir0/nqdir30 /nqdir0/qdir1/qdir20/qdir30 hasException = false; try { assertFalse(dfs.rename(new Path("/nqdir0/nqdir30"), tempPath)); } catch (NSQuotaExceededException e) { hasException = true; } assertTrue(hasException); // 14: Move /nqdir0/qdir1/qdir21 /nqdir0/qdir1/qdir20 assertTrue(dfs.rename(quotaDir3, quotaDir2)); c = dfs.getContentSummary(quotaDir1); assertEquals(c.getDirectoryCount(), 4); assertEquals(c.getQuota(), 6); c = dfs.getContentSummary(quotaDir2); assertEquals(c.getDirectoryCount(), 3); assertEquals(c.getQuota(), 7); tempPath = new Path(quotaDir2, "qdir21"); c = dfs.getContentSummary(tempPath); assertEquals(c.getDirectoryCount(), 1); assertEquals(c.getQuota(), 2); // 15: Delete /nqdir0/qdir1/qdir20/qdir21 dfs.delete(tempPath, true); c = dfs.getContentSummary(quotaDir2); assertEquals(c.getDirectoryCount(), 2); assertEquals(c.getQuota(), 7); c = dfs.getContentSummary(quotaDir1); assertEquals(c.getDirectoryCount(), 3); assertEquals(c.getQuota(), 6); // 16: Move /nqdir0/qdir30 /nqdir0/qdir1/qdir20 assertTrue(dfs.rename(new Path("/nqdir0/nqdir30"), quotaDir2)); c = dfs.getContentSummary(quotaDir2); assertEquals(c.getDirectoryCount(), 5); assertEquals(c.getQuota(), 7); c = dfs.getContentSummary(quotaDir1); assertEquals(c.getDirectoryCount(), 6); assertEquals(c.getQuota(), 6); } finally { cluster.shutdown(); }
5353 -1009652373apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None 10: Move /nqdir0/qdir1/qdir20/nqdir30 to /nqdir0/qdir1/qdir21 SATD_ADDED testNamespaceCommands() public void testNamespaceCommands() throws Exception final Configuration conf = new HdfsConfiguration(); final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); final FileSystem fs = cluster.getFileSystem(); assertTrue("Not a HDFS: " + fs.getUri(), fs instanceof DistributedFileSystem); final DistributedFileSystem dfs = (DistributedFileSystem) fs; try { // 1: create directory /nqdir0/qdir1/qdir20/nqdir30 assertTrue(dfs.mkdirs(new Path("/nqdir0/qdir1/qdir20/nqdir30"))); // 2: set the quota of /nqdir0/qdir1 to be 6 final Path quotaDir1 = new Path("/nqdir0/qdir1"); dfs.setQuota(quotaDir1, 6, FSConstants.QUOTA_DONT_SET); ContentSummary c = dfs.getContentSummary(quotaDir1); assertEquals(c.getDirectoryCount(), 3); assertEquals(c.getQuota(), 6); // 3: set the quota of /nqdir0/qdir1/qdir20 to be 7 final Path quotaDir2 = new Path("/nqdir0/qdir1/qdir20"); dfs.setQuota(quotaDir2, 7, FSConstants.QUOTA_DONT_SET); c = dfs.getContentSummary(quotaDir2); assertEquals(c.getDirectoryCount(), 2); assertEquals(c.getQuota(), 7); // 4: Create directory /nqdir0/qdir1/qdir21 and set its quota to 2 final Path quotaDir3 = new Path("/nqdir0/qdir1/qdir21"); assertTrue(dfs.mkdirs(quotaDir3)); dfs.setQuota(quotaDir3, 2, FSConstants.QUOTA_DONT_SET); c = dfs.getContentSummary(quotaDir3); assertEquals(c.getDirectoryCount(), 1); assertEquals(c.getQuota(), 2); // 5: Create directory /nqdir0/qdir1/qdir21/nqdir32 Path tempPath = new Path(quotaDir3, "nqdir32"); assertTrue(dfs.mkdirs(tempPath)); c = dfs.getContentSummary(quotaDir3); assertEquals(c.getDirectoryCount(), 2); assertEquals(c.getQuota(), 2); // 6: Create directory /nqdir0/qdir1/qdir21/nqdir33 tempPath = new Path(quotaDir3, "nqdir33"); boolean hasException = false; try { assertFalse(dfs.mkdirs(tempPath)); } catch (NSQuotaExceededException e) { hasException = true; } assertTrue(hasException); c = dfs.getContentSummary(quotaDir3); assertEquals(c.getDirectoryCount(), 2); assertEquals(c.getQuota(), 2); // 7: Create directory /nqdir0/qdir1/qdir20/nqdir31 tempPath = new Path(quotaDir2, "nqdir31"); assertTrue(dfs.mkdirs(tempPath)); c = dfs.getContentSummary(quotaDir2); assertEquals(c.getDirectoryCount(), 3); assertEquals(c.getQuota(), 7); c = dfs.getContentSummary(quotaDir1); assertEquals(c.getDirectoryCount(), 6); assertEquals(c.getQuota(), 6); // 8: Create directory /nqdir0/qdir1/qdir20/nqdir33 tempPath = new Path(quotaDir2, "nqdir33"); hasException = false; try { assertFalse(dfs.mkdirs(tempPath)); } catch (NSQuotaExceededException e) { hasException = true; } assertTrue(hasException); // 9: Move /nqdir0/qdir1/qdir21/nqdir32 /nqdir0/qdir1/qdir20/nqdir30 tempPath = new Path(quotaDir2, "nqdir30"); dfs.rename(new Path(quotaDir3, "nqdir32"), tempPath); c = dfs.getContentSummary(quotaDir2); assertEquals(c.getDirectoryCount(), 4); assertEquals(c.getQuota(), 7); c = dfs.getContentSummary(quotaDir1); assertEquals(c.getDirectoryCount(), 6); assertEquals(c.getQuota(), 6); // 10: Move /nqdir0/qdir1/qdir20/nqdir30 to /nqdir0/qdir1/qdir21 hasException = false; try { assertFalse(dfs.rename(tempPath, quotaDir3)); } catch (NSQuotaExceededException e) { hasException = true; } assertTrue(hasException); assertTrue(dfs.exists(tempPath)); assertFalse(dfs.exists(new Path(quotaDir3, "nqdir30"))); // 10.a: Rename /nqdir0/qdir1/qdir20/nqdir30 to /nqdir0/qdir1/qdir21/nqdir32 hasException = false; try { assertFalse(dfs.rename(tempPath, new Path(quotaDir3, "nqdir32"))); } catch (QuotaExceededException e) { hasException = true; } assertTrue(hasException); assertTrue(dfs.exists(tempPath)); assertFalse(dfs.exists(new Path(quotaDir3, "nqdir32"))); // 11: Move /nqdir0/qdir1/qdir20/nqdir30 to /nqdir0 assertTrue(dfs.rename(tempPath, new Path("/nqdir0"))); c = dfs.getContentSummary(quotaDir2); assertEquals(c.getDirectoryCount(), 2); assertEquals(c.getQuota(), 7); c = dfs.getContentSummary(quotaDir1); assertEquals(c.getDirectoryCount(), 4); assertEquals(c.getQuota(), 6); // 12: Create directory /nqdir0/nqdir30/nqdir33 assertTrue(dfs.mkdirs(new Path("/nqdir0/nqdir30/nqdir33"))); // 13: Move /nqdir0/nqdir30 /nqdir0/qdir1/qdir20/qdir30 hasException = false; try { assertFalse(dfs.rename(new Path("/nqdir0/nqdir30"), tempPath)); } catch (NSQuotaExceededException e) { hasException = true; } assertTrue(hasException); // 14: Move /nqdir0/qdir1/qdir21 /nqdir0/qdir1/qdir20 assertTrue(dfs.rename(quotaDir3, quotaDir2)); c = dfs.getContentSummary(quotaDir1); assertEquals(c.getDirectoryCount(), 4); assertEquals(c.getQuota(), 6); c = dfs.getContentSummary(quotaDir2); assertEquals(c.getDirectoryCount(), 3); assertEquals(c.getQuota(), 7); tempPath = new Path(quotaDir2, "qdir21"); c = dfs.getContentSummary(tempPath); assertEquals(c.getDirectoryCount(), 1); assertEquals(c.getQuota(), 2); // 15: Delete /nqdir0/qdir1/qdir20/qdir21 dfs.delete(tempPath, true); c = dfs.getContentSummary(quotaDir2); assertEquals(c.getDirectoryCount(), 2); assertEquals(c.getQuota(), 7); c = dfs.getContentSummary(quotaDir1); assertEquals(c.getDirectoryCount(), 3); assertEquals(c.getQuota(), 6); // 16: Move /nqdir0/qdir30 /nqdir0/qdir1/qdir20 assertTrue(dfs.rename(new Path("/nqdir0/nqdir30"), quotaDir2)); c = dfs.getContentSummary(quotaDir2); assertEquals(c.getDirectoryCount(), 5); assertEquals(c.getQuota(), 7); c = dfs.getContentSummary(quotaDir1); assertEquals(c.getDirectoryCount(), 6); assertEquals(c.getQuota(), 6); } finally { cluster.shutdown(); }
5352 -1009652374apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None 9: Move /nqdir0/qdir1/qdir21/nqdir32 /nqdir0/qdir1/qdir20/nqdir30 SATD_ADDED testNamespaceCommands() public void testNamespaceCommands() throws Exception final Configuration conf = new HdfsConfiguration(); final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); final FileSystem fs = cluster.getFileSystem(); assertTrue("Not a HDFS: " + fs.getUri(), fs instanceof DistributedFileSystem); final DistributedFileSystem dfs = (DistributedFileSystem) fs; try { // 1: create directory /nqdir0/qdir1/qdir20/nqdir30 assertTrue(dfs.mkdirs(new Path("/nqdir0/qdir1/qdir20/nqdir30"))); // 2: set the quota of /nqdir0/qdir1 to be 6 final Path quotaDir1 = new Path("/nqdir0/qdir1"); dfs.setQuota(quotaDir1, 6, FSConstants.QUOTA_DONT_SET); ContentSummary c = dfs.getContentSummary(quotaDir1); assertEquals(c.getDirectoryCount(), 3); assertEquals(c.getQuota(), 6); // 3: set the quota of /nqdir0/qdir1/qdir20 to be 7 final Path quotaDir2 = new Path("/nqdir0/qdir1/qdir20"); dfs.setQuota(quotaDir2, 7, FSConstants.QUOTA_DONT_SET); c = dfs.getContentSummary(quotaDir2); assertEquals(c.getDirectoryCount(), 2); assertEquals(c.getQuota(), 7); // 4: Create directory /nqdir0/qdir1/qdir21 and set its quota to 2 final Path quotaDir3 = new Path("/nqdir0/qdir1/qdir21"); assertTrue(dfs.mkdirs(quotaDir3)); dfs.setQuota(quotaDir3, 2, FSConstants.QUOTA_DONT_SET); c = dfs.getContentSummary(quotaDir3); assertEquals(c.getDirectoryCount(), 1); assertEquals(c.getQuota(), 2); // 5: Create directory /nqdir0/qdir1/qdir21/nqdir32 Path tempPath = new Path(quotaDir3, "nqdir32"); assertTrue(dfs.mkdirs(tempPath)); c = dfs.getContentSummary(quotaDir3); assertEquals(c.getDirectoryCount(), 2); assertEquals(c.getQuota(), 2); // 6: Create directory /nqdir0/qdir1/qdir21/nqdir33 tempPath = new Path(quotaDir3, "nqdir33"); boolean hasException = false; try { assertFalse(dfs.mkdirs(tempPath)); } catch (NSQuotaExceededException e) { hasException = true; } assertTrue(hasException); c = dfs.getContentSummary(quotaDir3); assertEquals(c.getDirectoryCount(), 2); assertEquals(c.getQuota(), 2); // 7: Create directory /nqdir0/qdir1/qdir20/nqdir31 tempPath = new Path(quotaDir2, "nqdir31"); assertTrue(dfs.mkdirs(tempPath)); c = dfs.getContentSummary(quotaDir2); assertEquals(c.getDirectoryCount(), 3); assertEquals(c.getQuota(), 7); c = dfs.getContentSummary(quotaDir1); assertEquals(c.getDirectoryCount(), 6); assertEquals(c.getQuota(), 6); // 8: Create directory /nqdir0/qdir1/qdir20/nqdir33 tempPath = new Path(quotaDir2, "nqdir33"); hasException = false; try { assertFalse(dfs.mkdirs(tempPath)); } catch (NSQuotaExceededException e) { hasException = true; } assertTrue(hasException); // 9: Move /nqdir0/qdir1/qdir21/nqdir32 /nqdir0/qdir1/qdir20/nqdir30 tempPath = new Path(quotaDir2, "nqdir30"); dfs.rename(new Path(quotaDir3, "nqdir32"), tempPath); c = dfs.getContentSummary(quotaDir2); assertEquals(c.getDirectoryCount(), 4); assertEquals(c.getQuota(), 7); c = dfs.getContentSummary(quotaDir1); assertEquals(c.getDirectoryCount(), 6); assertEquals(c.getQuota(), 6); // 10: Move /nqdir0/qdir1/qdir20/nqdir30 to /nqdir0/qdir1/qdir21 hasException = false; try { assertFalse(dfs.rename(tempPath, quotaDir3)); } catch (NSQuotaExceededException e) { hasException = true; } assertTrue(hasException); assertTrue(dfs.exists(tempPath)); assertFalse(dfs.exists(new Path(quotaDir3, "nqdir30"))); // 10.a: Rename /nqdir0/qdir1/qdir20/nqdir30 to /nqdir0/qdir1/qdir21/nqdir32 hasException = false; try { assertFalse(dfs.rename(tempPath, new Path(quotaDir3, "nqdir32"))); } catch (QuotaExceededException e) { hasException = true; } assertTrue(hasException); assertTrue(dfs.exists(tempPath)); assertFalse(dfs.exists(new Path(quotaDir3, "nqdir32"))); // 11: Move /nqdir0/qdir1/qdir20/nqdir30 to /nqdir0 assertTrue(dfs.rename(tempPath, new Path("/nqdir0"))); c = dfs.getContentSummary(quotaDir2); assertEquals(c.getDirectoryCount(), 2); assertEquals(c.getQuota(), 7); c = dfs.getContentSummary(quotaDir1); assertEquals(c.getDirectoryCount(), 4); assertEquals(c.getQuota(), 6); // 12: Create directory /nqdir0/nqdir30/nqdir33 assertTrue(dfs.mkdirs(new Path("/nqdir0/nqdir30/nqdir33"))); // 13: Move /nqdir0/nqdir30 /nqdir0/qdir1/qdir20/qdir30 hasException = false; try { assertFalse(dfs.rename(new Path("/nqdir0/nqdir30"), tempPath)); } catch (NSQuotaExceededException e) { hasException = true; } assertTrue(hasException); // 14: Move /nqdir0/qdir1/qdir21 /nqdir0/qdir1/qdir20 assertTrue(dfs.rename(quotaDir3, quotaDir2)); c = dfs.getContentSummary(quotaDir1); assertEquals(c.getDirectoryCount(), 4); assertEquals(c.getQuota(), 6); c = dfs.getContentSummary(quotaDir2); assertEquals(c.getDirectoryCount(), 3); assertEquals(c.getQuota(), 7); tempPath = new Path(quotaDir2, "qdir21"); c = dfs.getContentSummary(tempPath); assertEquals(c.getDirectoryCount(), 1); assertEquals(c.getQuota(), 2); // 15: Delete /nqdir0/qdir1/qdir20/qdir21 dfs.delete(tempPath, true); c = dfs.getContentSummary(quotaDir2); assertEquals(c.getDirectoryCount(), 2); assertEquals(c.getQuota(), 7); c = dfs.getContentSummary(quotaDir1); assertEquals(c.getDirectoryCount(), 3); assertEquals(c.getQuota(), 6); // 16: Move /nqdir0/qdir30 /nqdir0/qdir1/qdir20 assertTrue(dfs.rename(new Path("/nqdir0/nqdir30"), quotaDir2)); c = dfs.getContentSummary(quotaDir2); assertEquals(c.getDirectoryCount(), 5); assertEquals(c.getQuota(), 7); c = dfs.getContentSummary(quotaDir1); assertEquals(c.getDirectoryCount(), 6); assertEquals(c.getQuota(), 6); } finally { cluster.shutdown(); }
5351 -1009652375apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None TODO: figure out why length would be 0 SATD_ADDED getProgress() public float getProgress() throws IOException // TODO: figure out why length would be 0 if (length == 0) return 1; return pos / length;
5350 -1009652376apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None As we already know current Capacity percent of this queue make children distribute unconfigured Capacity. SATD_ADDED distributeUnConfiguredCapacity() void distributeUnConfiguredCapacity() List unConfiguredQueues = new ArrayList(); float totalCapacity = 0; for (AbstractQueue q : children) { if (q.qsc.getCapacityPercent() == -1) { // Add into unConfigured queue. unConfiguredQueues.add(q); } else { // If capacity is set , then add that to totalCapacity. LOG.info(" the capacity percent of the queue " + q.getName() + " is " + "" + q.qsc.getCapacityPercent()); totalCapacity += q.qsc.getCapacityPercent(); // As we already know current Capacity percent of this queue // make children distribute unconfigured Capacity. q.distributeUnConfiguredCapacity(); } } if (!unConfiguredQueues.isEmpty()) { LOG.info("Total capacity to be distributed among the others are " + "" + (100 - totalCapacity)); // We have list of queues at this level which are unconfigured. // 100 - totalCapacity is the capacity remaining. // Divide it equally among all the un configured queues. float capacityShare = (100 - totalCapacity) / unConfiguredQueues.size(); // We dont have to check for 100 - totalCapacity being -ve , as // we already do it while loading. for (AbstractQueue q : unConfiguredQueues) { if (q.qsc.getMaxCapacityPercent() > 0) { if (q.qsc.getMaxCapacityPercent() < capacityShare) { throw new IllegalStateException(" Capacity share (" + capacityShare + ")for unconfigured queue " + q.getName() + " is greater than its maximum-capacity percentage " + q.qsc.getMaxCapacityPercent()); } } q.qsc.setCapacityPercent(capacityShare); LOG.info("Capacity share for un configured queue " + q.getName() + "" + " is " + capacityShare); // we have q's capacity now. // make children also distribute it among themselves. q.distributeUnConfiguredCapacity(); } }
5349 -1009652377apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None delete output -- should we really do this? SATD_ADDED run(String[]) public int run(String[] args) throws Exception String version = "MRBenchmark.0.0.2"; System.out.println(version); String usage = "Usage: mrbench " + "[-baseDir ] " + "[-jar ] " + "[-numRuns ] " + "[-maps ] " + "[-reduces ] " + "[-inputLines ] " + "[-inputType ] " + "[-verbose]"; String jarFile = null; int inputLines = 1; int numRuns = 1; int numMaps = 2; int numReduces = 1; boolean verbose = false; Order inputSortOrder = Order.ASCENDING; for (int i = 0; i < args.length; i++) { // parse command line if (args[i].equals("-jar")) { jarFile = args[++i]; } else if (args[i].equals("-numRuns")) { numRuns = Integer.parseInt(args[++i]); } else if (args[i].equals("-baseDir")) { BASE_DIR = new Path(args[++i]); } else if (args[i].equals("-maps")) { numMaps = Integer.parseInt(args[++i]); } else if (args[i].equals("-reduces")) { numReduces = Integer.parseInt(args[++i]); } else if (args[i].equals("-inputLines")) { inputLines = Integer.parseInt(args[++i]); } else if (args[i].equals("-inputType")) { String s = args[++i]; if (s.equalsIgnoreCase("ascending")) { inputSortOrder = Order.ASCENDING; } else if (s.equalsIgnoreCase("descending")) { inputSortOrder = Order.DESCENDING; } else if (s.equalsIgnoreCase("random")) { inputSortOrder = Order.RANDOM; } else { inputSortOrder = null; } } else if (args[i].equals("-verbose")) { verbose = true; } else { System.err.println(usage); System.exit(-1); } } if (// verify args numRuns < 1 || numMaps < 1 || numReduces < 1 || inputLines < 0 || inputSortOrder == null) { System.err.println(usage); return -1; } JobConf jobConf = setupJob(numMaps, numReduces, jarFile); FileSystem fs = FileSystem.get(jobConf); Path inputFile = new Path(INPUT_DIR, "input_" + (new Random()).nextInt() + ".txt"); generateTextFile(fs, inputFile, inputLines, inputSortOrder); // setup test output directory fs.mkdirs(BASE_DIR); ArrayList execTimes = new ArrayList(); try { execTimes = runJobInSequence(jobConf, numRuns); } finally { // delete output -- should we really do this? fs.delete(BASE_DIR, true); } if (verbose) { // Print out a report System.out.println("Total MapReduce jobs executed: " + numRuns); System.out.println("Total lines of data per job: " + inputLines); System.out.println("Maps per job: " + numMaps); System.out.println("Reduces per job: " + numReduces); } int i = 0; long totalTime = 0; for (Long time : execTimes) { totalTime += time.longValue(); if (verbose) { System.out.println("Total milliseconds for task: " + (++i) + " = " + time); } } long avgTime = totalTime / numRuns; System.out.println("DataLines\tMaps\tReduces\tAvgTime (milliseconds)"); System.out.println(inputLines + "\t\t" + numMaps + "\t" + numReduces + "\t" + avgTime); return 0;
5348 -1009652378apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None divide the right into 5 31 bit chunks SATD_ADDED multiply(Unsigned16) void multiply(Unsigned16 b) // divide the left into 4 32 bit chunks long[] left = new long[4]; left[0] = lo8 & 0xffffffffl; left[1] = lo8 >>> 32; left[2] = hi8 & 0xffffffffl; left[3] = hi8 >>> 32; // divide the right into 5 31 bit chunks long[] right = new long[5]; right[0] = b.lo8 & 0x7fffffffl; right[1] = (b.lo8 >>> 31) & 0x7fffffffl; right[2] = (b.lo8 >>> 62) + ((b.hi8 & 0x1fffffffl) << 2); right[3] = (b.hi8 >>> 29) & 0x7fffffffl; right[4] = (b.hi8 >>> 60); // clear the cur value set(0); Unsigned16 tmp = new Unsigned16(); for (int l = 0; l < 4; ++l) { for (int r = 0; r < 5; ++r) { long prod = left[l] * right[r]; if (prod != 0) { int off = l * 32 + r * 31; tmp.set(prod); tmp.shiftLeft(off); add(tmp); } } }
5347 -1009652379apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None divide the left into 4 32 bit chunks SATD_ADDED multiply(Unsigned16) void multiply(Unsigned16 b) // divide the left into 4 32 bit chunks long[] left = new long[4]; left[0] = lo8 & 0xffffffffl; left[1] = lo8 >>> 32; left[2] = hi8 & 0xffffffffl; left[3] = hi8 >>> 32; // divide the right into 5 31 bit chunks long[] right = new long[5]; right[0] = b.lo8 & 0x7fffffffl; right[1] = (b.lo8 >>> 31) & 0x7fffffffl; right[2] = (b.lo8 >>> 62) + ((b.hi8 & 0x1fffffffl) << 2); right[3] = (b.hi8 >>> 29) & 0x7fffffffl; right[4] = (b.hi8 >>> 60); // clear the cur value set(0); Unsigned16 tmp = new Unsigned16(); for (int l = 0; l < 4; ++l) { for (int r = 0; r < 5; ++r) { long prod = left[l] * right[r]; if (prod != 0) { int off = l * 32 + r * 31; tmp.set(prod); tmp.shiftLeft(off); add(tmp); } } }
5346 -1009652380apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Move the offset to the start of the block. SATD_ADDED decodeFile(FileSystem, Path, FileSystem, Path, long, Path) public void decodeFile(FileSystem fs, Path srcFile, FileSystem parityFs, Path parityFile, long errorOffset, Path decodedFile) throws IOException LOG.info("Create " + decodedFile + " for error at " + srcFile + ":" + errorOffset); FileStatus srcStat = fs.getFileStatus(srcFile); long blockSize = srcStat.getBlockSize(); configureBuffers(blockSize); // Move the offset to the start of the block. errorOffset = (errorOffset / blockSize) * blockSize; // Create the decoded file. FSDataOutputStream out = fs.create(decodedFile, false, conf.getInt("io.file.buffer.size", 64 * 1024), srcStat.getReplication(), srcStat.getBlockSize()); // Open the source file. FSDataInputStream in = fs.open(srcFile, conf.getInt("io.file.buffer.size", 64 * 1024)); // Start copying data block-by-block. for (long offset = 0; offset < srcStat.getLen(); offset += blockSize) { long limit = Math.min(blockSize, srcStat.getLen() - offset); long bytesAlreadyCopied = 0; if (offset != errorOffset) { try { in = fs.open(srcFile, conf.getInt("io.file.buffer.size", 64 * 1024)); in.seek(offset); RaidUtils.copyBytes(in, out, readBufs[0], limit); assert (out.getPos() == offset + limit); LOG.info("Copied till " + out.getPos() + " from " + srcFile); continue; } catch (BlockMissingException e) { LOG.warn("Encountered BME at " + srcFile + ":" + offset); bytesAlreadyCopied = out.getPos() - offset; } catch (ChecksumException e) { LOG.warn("Encountered CE at " + srcFile + ":" + offset); bytesAlreadyCopied = out.getPos() - offset; } } // If we are here offset == errorOffset or we got an exception. // Recover the block starting at offset. fixErasedBlock(fs, srcFile, parityFs, parityFile, blockSize, offset, bytesAlreadyCopied, limit, out); } out.close(); try { fs.setOwner(decodedFile, srcStat.getOwner(), srcStat.getGroup()); fs.setPermission(decodedFile, srcStat.getPermission()); fs.setTimes(decodedFile, srcStat.getModificationTime(), srcStat.getAccessTime()); } catch (Exception exc) { LOG.warn("Didn't manage to copy meta information because of " + exc + " Ignoring..."); }
5345 -1009652381apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None set the output format to take care of the _temporary folder SATD_ADDED testSmallTrace() public void testSmallTrace() throws Exception performSingleTest("sample-job-tracker-logs.gz", "job-tracker-logs-topology-output", "job-tracker-logs-trace-output.gz");
5343 -1009652382apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Try operations as an authorized user, who is a queue administrator. SATD_ADDED verifyACLViewJob() private void verifyACLViewJob() throws IOException, InterruptedException // Set the job up. final Configuration myConf = mr.createJobConf(); myConf.set(MRJobConfig.JOB_ACL_VIEW_JOB, viewColleague); // Submit the job as user1 Job job = submitJobAsUser(myConf, jobSubmitter); final JobID jobId = job.getJobID(); // Try operations as an unauthorized user. verifyViewJobAsUnauthorizedUser(myConf, jobId, modifyColleague); // Try operations as an authorized user, who is part of view-job-acl. verifyViewJobAsAuthorizedUser(myConf, jobId, viewColleague); // Try operations as an authorized user, who is a queue administrator. verifyViewJobAsAuthorizedUser(myConf, jobId, qAdmin); // Clean up the job job.killJob();
5342 -1009652126apache/hadoopThomas Whitea732ab38047299c545cfc978e25f6b2871b5a4b3 None configure task jvm options if enabled this knob can be turned off if there is a mismatch between the target (simulation) cluster and the original cluster. Such a mismatch can result in job failures (due to memory issues) on the target (simulated) cluster. TODO If configured, scale the original task's JVM (heap related) options to suit the target (simulation) cluster SATD_ADDED initialValue() protected Formatter initialValue() final StringBuilder sb = new StringBuilder(JOB_NAME_PREFIX.length() + 6); sb.append(JOB_NAME_PREFIX); return new Formatter(sb);
5341 -1009652383apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Try operations as an authorized user, who is part of view-job-acl. SATD_ADDED verifyACLViewJob() private void verifyACLViewJob() throws IOException, InterruptedException // Set the job up. final Configuration myConf = mr.createJobConf(); myConf.set(MRJobConfig.JOB_ACL_VIEW_JOB, viewColleague); // Submit the job as user1 Job job = submitJobAsUser(myConf, jobSubmitter); final JobID jobId = job.getJobID(); // Try operations as an unauthorized user. verifyViewJobAsUnauthorizedUser(myConf, jobId, modifyColleague); // Try operations as an authorized user, who is part of view-job-acl. verifyViewJobAsAuthorizedUser(myConf, jobId, viewColleague); // Try operations as an authorized user, who is a queue administrator. verifyViewJobAsAuthorizedUser(myConf, jobId, qAdmin); // Clean up the job job.killJob();
5340 -1009652127apache/hadoopThomas Whitea732ab38047299c545cfc978e25f6b2871b5a4b3 None test whether the heap usage emulator achieves the desired target using desired calls to the underling core engine. SATD_ADDED testEmulationAccuracy(Configuration, FakeHeapUsageEmulatorCore, ResourceCalculatorPlugin, ResourceUsageMetrics, TotalHeapUsageEmulatorPlugin, long, long) private static void testEmulationAccuracy(Configuration conf, FakeHeapUsageEmulatorCore fakeCore, ResourceCalculatorPlugin monitor, ResourceUsageMetrics metrics, TotalHeapUsageEmulatorPlugin heapPlugin, long expectedTotalHeapUsageInMB, long expectedTotalNumCalls) throws Exception FakeProgressive fakeProgress = new FakeProgressive(); fakeCore.resetFake(); heapPlugin.initialize(conf, metrics, monitor, fakeProgress); int numLoops = 0; while (fakeProgress.getProgress() < 1) { ++numLoops; float progress = numLoops / 100.0F; fakeProgress.setProgress(progress); heapPlugin.emulate(); } // test if the resource plugin shows the expected usage assertEquals("Cumulative heap usage emulator plugin failed (total usage)!", expectedTotalHeapUsageInMB, fakeCore.getHeapUsageInMB(), 1L); // test if the resource plugin shows the expected num calls assertEquals("Cumulative heap usage emulator plugin failed (num calls)!", expectedTotalNumCalls, fakeCore.getNumCalls(), 0L);
5339 -1009652384apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Enable queue and job level authorization SATD_ADDED startCluster(boolean) private void startCluster(boolean reStart) throws Exception // Configure job queues String[] queueNames = { "default" }; createQueuesConfigFile(queueNames, new String[] { jobSubmitter }, new String[] { qAdmin }); JobConf conf = new JobConf(); // Enable queue and job level authorization conf.setBoolean(MRConfig.MR_ACLS_ENABLED, true); // Enable CompletedJobStore FileSystem fs = FileSystem.getLocal(conf); if (!reStart) { fs.delete(TEST_DIR, true); } conf.set(JTConfig.JT_PERSIST_JOBSTATUS_DIR, fs.makeQualified(TEST_DIR).toString()); conf.setBoolean(JTConfig.JT_PERSIST_JOBSTATUS, true); conf.set(JTConfig.JT_PERSIST_JOBSTATUS_HOURS, "1"); UserGroupInformation MR_UGI = UserGroupInformation.getLoginUser(); mr = new MiniMRCluster(0, 0, 1, "file:///", 1, null, null, MR_UGI, conf);
5338 -1009652128apache/hadoopThomas Whitea732ab38047299c545cfc978e25f6b2871b5a4b3 None test with invalid or missing resource usage value SATD_ADDED testTotalHeapUsageEmulatorPlugin() public void testTotalHeapUsageEmulatorPlugin() throws Exception Configuration conf = new Configuration(); // set the dummy resource calculator for testing ResourceCalculatorPlugin monitor = new DummyResourceCalculatorPlugin(); // 1GB long maxHeapUsage = 1024 * TotalHeapUsageEmulatorPlugin.ONE_MB; conf.setLong(DummyResourceCalculatorPlugin.MAXPMEM_TESTING_PROPERTY, maxHeapUsage); monitor.setConf(conf); // no buffer to be reserved conf.setFloat(TotalHeapUsageEmulatorPlugin.MIN_HEAP_FREE_RATIO, 0F); // only 1 call to be made per cycle conf.setFloat(TotalHeapUsageEmulatorPlugin.HEAP_LOAD_RATIO, 1F); // 200mb long targetHeapUsageInMB = 200; // fake progress indicator FakeProgressive fakeProgress = new FakeProgressive(); // fake heap usage generator FakeHeapUsageEmulatorCore fakeCore = new FakeHeapUsageEmulatorCore(); // a heap usage emulator with fake core FakeHeapUsageEmulatorPlugin heapPlugin = new FakeHeapUsageEmulatorPlugin(fakeCore); // test with invalid or missing resource usage value ResourceUsageMetrics invalidUsage = TestResourceUsageEmulators.createMetrics(0); heapPlugin.initialize(conf, invalidUsage, null, null); // test if disabled heap emulation plugin's emulate() call is a no-operation // this will test if the emulation plugin is disabled or not int numCallsPre = fakeCore.getNumCalls(); long heapUsagePre = fakeCore.getHeapUsageInMB(); heapPlugin.emulate(); int numCallsPost = fakeCore.getNumCalls(); long heapUsagePost = fakeCore.getHeapUsageInMB(); // test if no calls are made heap usage emulator core assertEquals("Disabled heap usage emulation plugin works!", numCallsPre, numCallsPost); // test if no calls are made heap usage emulator core assertEquals("Disabled heap usage emulation plugin works!", heapUsagePre, heapUsagePost); // test with wrong/invalid configuration Boolean failed = null; invalidUsage = TestResourceUsageEmulators.createMetrics(maxHeapUsage + TotalHeapUsageEmulatorPlugin.ONE_MB); try { heapPlugin.initialize(conf, invalidUsage, monitor, null); failed = false; } catch (Exception e) { failed = true; } assertNotNull("Fail case failure!", failed); assertTrue("Expected failure!", failed); // test with valid resource usage value ResourceUsageMetrics metrics = TestResourceUsageEmulators.createMetrics(targetHeapUsageInMB * TotalHeapUsageEmulatorPlugin.ONE_MB); // test with default emulation interval // in every interval, the emulator will add 100% of the expected usage // (since gridmix.emulators.resource-usage.heap.load-ratio=1) // so at 10%, emulator will add 10% (difference), at 20% it will add 10% ... // So to emulate 200MB, it will add // 20mb + 20mb + 20mb + 20mb + .. = 200mb testEmulationAccuracy(conf, fakeCore, monitor, metrics, heapPlugin, 200, 10); // test with custom value for emulation interval of 20% conf.setFloat(TotalHeapUsageEmulatorPlugin.HEAP_EMULATION_PROGRESS_INTERVAL, 0.2F); // 40mb + 40mb + 40mb + 40mb + 40mb = 200mb testEmulationAccuracy(conf, fakeCore, monitor, metrics, heapPlugin, 200, 5); // test with custom value of free heap ratio and load ratio = 1 conf.setFloat(TotalHeapUsageEmulatorPlugin.HEAP_LOAD_RATIO, 1F); conf.setFloat(TotalHeapUsageEmulatorPlugin.MIN_HEAP_FREE_RATIO, 0.5F); // 40mb + 0mb + 80mb + 0mb + 0mb = 120mb testEmulationAccuracy(conf, fakeCore, monitor, metrics, heapPlugin, 120, 2); // test with custom value of heap load ratio and min free heap ratio = 0 conf.setFloat(TotalHeapUsageEmulatorPlugin.HEAP_LOAD_RATIO, 0.5F); conf.setFloat(TotalHeapUsageEmulatorPlugin.MIN_HEAP_FREE_RATIO, 0F); // 20mb (call#1) + 20mb (call#1) + 20mb (call#2) + 20mb (call#2) +.. = 200mb testEmulationAccuracy(conf, fakeCore, monitor, metrics, heapPlugin, 200, 10); // test with custom value of free heap ratio = 0.3 and load ratio = 0.5 conf.setFloat(TotalHeapUsageEmulatorPlugin.MIN_HEAP_FREE_RATIO, 0.25F); conf.setFloat(TotalHeapUsageEmulatorPlugin.HEAP_LOAD_RATIO, 0.5F); // 20mb (call#1) + 20mb (call#1) + 30mb (call#2) + 0mb (call#2) // + 30mb (call#3) + 0mb (call#3) + 35mb (call#4) + 0mb (call#4) // + 37mb (call#5) + 0mb (call#5) = 162mb testEmulationAccuracy(conf, fakeCore, monitor, metrics, heapPlugin, 162, 6); // test if emulation interval boundary is respected // initialize fakeProgress = new FakeProgressive(); conf.setFloat(TotalHeapUsageEmulatorPlugin.MIN_HEAP_FREE_RATIO, 0F); conf.setFloat(TotalHeapUsageEmulatorPlugin.HEAP_LOAD_RATIO, 1F); conf.setFloat(TotalHeapUsageEmulatorPlugin.HEAP_EMULATION_PROGRESS_INTERVAL, 0.25F); heapPlugin.initialize(conf, metrics, monitor, fakeProgress); fakeCore.resetFake(); // take a snapshot after the initialization long initHeapUsage = fakeCore.getHeapUsageInMB(); long initNumCallsUsage = fakeCore.getNumCalls(); // test with 0 progress testEmulationBoundary(0F, fakeCore, fakeProgress, heapPlugin, initHeapUsage, initNumCallsUsage, "[no-op, 0 progress]"); // test with 24% progress testEmulationBoundary(0.24F, fakeCore, fakeProgress, heapPlugin, initHeapUsage, initNumCallsUsage, "[no-op, 24% progress]"); // test with 25% progress testEmulationBoundary(0.25F, fakeCore, fakeProgress, heapPlugin, targetHeapUsageInMB / 4, 1, "[op, 25% progress]"); // test with 80% progress testEmulationBoundary(0.80F, fakeCore, fakeProgress, heapPlugin, (targetHeapUsageInMB * 4) / 5, 2, "[op, 80% progress]"); // now test if the final call with 100% progress ramps up the heap usage testEmulationBoundary(1F, fakeCore, fakeProgress, heapPlugin, targetHeapUsageInMB, 3, "[op, 100% progress]");
5337 -1009652385apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Hack for local FS that does not have the concept of a 'mounting point' SATD_ADDED launchWordCount(JobConf, String, int, int) private String launchWordCount(JobConf conf, String input, int numMaps, int numReduces) throws IOException Path inDir = new Path("testing/wc/input"); Path outDir = new Path("testing/wc/output"); // Hack for local FS that does not have the concept of a 'mounting point' if (isLocalFS()) { String localPathRoot = System.getProperty("test.build.data", "/tmp").toString().replace(' ', '+'); ; inDir = new Path(localPathRoot, inDir); outDir = new Path(localPathRoot, outDir); } FileSystem fs = FileSystem.get(conf); fs.delete(outDir, true); if (!fs.mkdirs(inDir)) { throw new IOException("Mkdirs failed to create " + inDir.toString()); } { DataOutputStream file = fs.create(new Path(inDir, "part-0")); file.writeBytes(input); file.close(); } conf.setJobName("wordcount"); conf.setInputFormat(TextInputFormat.class); // the keys are words (strings) conf.setOutputKeyClass(Text.class); // the values are counts (ints) conf.setOutputValueClass(IntWritable.class); conf.setMapperClass(WordCount.MapClass.class); conf.setCombinerClass(WordCount.Reduce.class); conf.setReducerClass(WordCount.Reduce.class); FileInputFormat.setInputPaths(conf, inDir); FileOutputFormat.setOutputPath(conf, outDir); conf.setNumMapTasks(numMaps); conf.setNumReduceTasks(numReduces); JobClient.runJob(conf); return MapReduceTestUtil.readOutput(outDir, conf);
5336 -1009652386apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Hack for local FS that does not have the concept of a 'mounting point' SATD_ADDED testMR() public void testMR() throws Exception System.out.println(launchWordCount(this.createJobConf(), "a b c d e f g h", 1, 1)); Thread.sleep(2000); assertEquals(2, NotificationServlet.counter); Path inDir = new Path("notificationjob/input"); Path outDir = new Path("notificationjob/output"); // Hack for local FS that does not have the concept of a 'mounting point' if (isLocalFS()) { String localPathRoot = System.getProperty("test.build.data", "/tmp").toString().replace(' ', '+'); ; inDir = new Path(localPathRoot, inDir); outDir = new Path(localPathRoot, outDir); } // run a job with KILLED status System.out.println(UtilsForTests.runJobKill(this.createJobConf(), inDir, outDir).getID()); Thread.sleep(2000); assertEquals(4, NotificationServlet.counter); // run a job with FAILED status System.out.println(UtilsForTests.runJobFail(this.createJobConf(), inDir, outDir).getID()); Thread.sleep(2000); assertEquals(6, NotificationServlet.counter);
5335 -1009652387apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None get a non-exist reduce task attempt, expect a made-up task attempt TODO fill in test case SATD_ADDED testFirstJob() public void testFirstJob() // 20th job seems reasonable: "totalMaps":329,"totalReduces":101 // successful map: 80 node-local, 196 rack-local, 53 rack-remote, 2 unknown // failed map: 0-0-0-1 // successful reduce: 99 failed reduce: 13 // map attempts to success -- 0.9969879518072289, 0.0030120481927710845, JobStory job = jobStories.get(0); assertEquals(1, job.getNumberMaps()); assertEquals(1, job.getNumberReduces()); // get splits TaskAttemptInfo taInfo = null; long expectedRuntime = 2423; // get a succeeded map task attempt, expect the exact same task attempt taInfo = job.getMapTaskAttemptInfoAdjusted(14, 0, 1); assertEquals(expectedRuntime, taInfo.getRuntime()); assertEquals(State.SUCCEEDED, taInfo.getRunState()); // get a succeeded map attempt, but reschedule with different locality. taInfo = job.getMapTaskAttemptInfoAdjusted(14, 0, 2); assertEquals(State.SUCCEEDED, taInfo.getRunState()); taInfo = job.getMapTaskAttemptInfoAdjusted(14, 0, 0); assertEquals(State.SUCCEEDED, taInfo.getRunState()); expectedRuntime = 97502; // get a succeeded reduce task attempt, expect the exact same task attempt taInfo = job.getTaskAttemptInfo(TaskType.REDUCE, 14, 0); assertEquals(State.SUCCEEDED, taInfo.getRunState()); // get a failed reduce task attempt, expect the exact same task attempt taInfo = job.getTaskAttemptInfo(TaskType.REDUCE, 14, 0); assertEquals(State.SUCCEEDED, taInfo.getRunState()); // get a non-exist reduce task attempt, expect a made-up task attempt // TODO fill in test case
5334 -1009652388apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Poll the Namenode (once every 5 minutes) to find the size of the pending edit log. 5 minutes SATD_ADDED doWork() public void doWork() // // Poll the Namenode (once every 5 minutes) to find the size of the // pending edit log. // // 5 minutes long period = 5 * 60; if (checkpointPeriod < period) { period = checkpointPeriod; } while (shouldRun) { try { Thread.sleep(1000 * period); } catch (InterruptedException ie) { // do nothing } if (!shouldRun) { break; } try { // We may have lost our ticket since last checkpoint, log in again, just in case if (UserGroupInformation.isSecurityEnabled()) UserGroupInformation.getCurrentUser().reloginFromKeytab(); long now = System.currentTimeMillis(); long size = namenode.getEditLogSize(); if (size >= checkpointSize || now >= lastCheckpointTime + 1000 * checkpointPeriod) { doCheckpoint(); lastCheckpointTime = now; } } catch (IOException e) { LOG.error("Exception in doCheckpoint: "); LOG.error(StringUtils.stringifyException(e)); e.printStackTrace(); checkpointImage.getStorage().imageDigest = null; } catch (Throwable e) { LOG.error("Throwable Exception in doCheckpoint: "); LOG.error(StringUtils.stringifyException(e)); e.printStackTrace(); Runtime.getRuntime().exit(-1); } }
5333 -1009652389apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Calculate the best available replica state. SATD_ADDED syncBlock(RecoveringBlock, List) void syncBlock(RecoveringBlock rBlock, List syncList) throws IOException ExtendedBlock block = rBlock.getBlock(); DatanodeProtocol nn = getBPNamenode(block.getBlockPoolId()); long recoveryId = rBlock.getNewGenerationStamp(); if (LOG.isDebugEnabled()) { LOG.debug("block=" + block + ", (length=" + block.getNumBytes() + "), syncList=" + syncList); } // syncList.isEmpty() means that all data-nodes do not have the block // or their replicas have 0 length. // The block can be deleted. if (syncList.isEmpty()) { nn.commitBlockSynchronization(block, recoveryId, 0, true, true, DatanodeID.EMPTY_ARRAY); return; } // Calculate the best available replica state. ReplicaState bestState = ReplicaState.RWR; long finalizedLength = -1; for (BlockRecord r : syncList) { assert r.rInfo.getNumBytes() > 0 : "zero length replica"; ReplicaState rState = r.rInfo.getOriginalReplicaState(); if (rState.getValue() < bestState.getValue()) bestState = rState; if (rState == ReplicaState.FINALIZED) { if (finalizedLength > 0 && finalizedLength != r.rInfo.getNumBytes()) throw new IOException("Inconsistent size of finalized replicas. " + "Replica " + r.rInfo + " expected size: " + finalizedLength); finalizedLength = r.rInfo.getNumBytes(); } } // Calculate list of nodes that will participate in the recovery // and the new block size List participatingList = new ArrayList(); final ExtendedBlock newBlock = new ExtendedBlock(block.getBlockPoolId(), block.getBlockId(), -1, recoveryId); switch(bestState) { case FINALIZED: assert finalizedLength > 0 : "finalizedLength is not positive"; for (BlockRecord r : syncList) { ReplicaState rState = r.rInfo.getOriginalReplicaState(); if (rState == ReplicaState.FINALIZED || rState == ReplicaState.RBW && r.rInfo.getNumBytes() == finalizedLength) participatingList.add(r); } newBlock.setNumBytes(finalizedLength); break; case RBW: case RWR: long minLength = Long.MAX_VALUE; for (BlockRecord r : syncList) { ReplicaState rState = r.rInfo.getOriginalReplicaState(); if (rState == bestState) { minLength = Math.min(minLength, r.rInfo.getNumBytes()); participatingList.add(r); } } newBlock.setNumBytes(minLength); break; case RUR: case TEMPORARY: assert false : "bad replica state: " + bestState; } List failedList = new ArrayList(); List successList = new ArrayList(); for (BlockRecord r : participatingList) { try { ExtendedBlock reply = r.datanode.updateReplicaUnderRecovery(new ExtendedBlock(newBlock.getBlockPoolId(), r.rInfo), recoveryId, newBlock.getNumBytes()); assert reply.equals(newBlock) && reply.getNumBytes() == newBlock.getNumBytes() : "Updated replica must be the same as the new block."; successList.add(r.id); } catch (IOException e) { InterDatanodeProtocol.LOG.warn("Failed to updateBlock (newblock=" + newBlock + ", datanode=" + r.id + ")", e); failedList.add(r.id); } } // If any of the data-nodes failed, the recovery fails, because // we never know the actual state of the replica on failed data-nodes. // The recovery should be started over. if (!failedList.isEmpty()) { StringBuilder b = new StringBuilder(); for (DatanodeID id : failedList) { b.append("\n " + id); } throw new IOException("Cannot recover " + block + ", the following " + failedList.size() + " data-nodes failed {" + b + "\n}"); } // Notify the name-node about successfully recovered replicas. DatanodeID[] nlist = successList.toArray(new DatanodeID[successList.size()]); nn.commitBlockSynchronization(block, newBlock.getGenerationStamp(), newBlock.getNumBytes(), true, false, nlist);
5332 -1009652390apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Return \"DS-randInt-ipaddr-currentTimeMillis\" It is considered extermely rare for all these numbers to match on a different machine accidentally for the following a) SecureRandom(INT_MAX) is pretty much random (1 in 2 billion), and b) Good chance ip address would be different, and c) Even on the same machine, Datanode is designed to use different ports. d) Good chance that these are started at different times. For a confict to occur all the 4 above have to match!. The format of this string can be changed anytime in future without affecting its functionality. SATD_ADDED createSocketAddr(String) public static InetSocketAddress createSocketAddr(String target) throws IOException return NetUtils.createSocketAddr(target);
5331 -1009652391apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None it would have been better to pass storage as a parameter to constructor below - need to augment ReflectionUtils used below. SATD_ADDED createSocketAddr(String) public static InetSocketAddress createSocketAddr(String target) throws IOException return NetUtils.createSocketAddr(target);
5330 -1009652392apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None One common reason is that NameNode could be in safe mode. Should we keep on retrying in that case? SATD_ADDED createSocketAddr(String) public static InetSocketAddress createSocketAddr(String target) throws IOException return NetUtils.createSocketAddr(target);
5329 -1009652393apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None set service-level authorization security policy SATD_ADDED createSocketAddr(String) public static InetSocketAddress createSocketAddr(String target) throws IOException return NetUtils.createSocketAddr(target);
5328 -1009652394apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None We generate leaves for a single split point as well as for no split points. SATD_ADDED buildTrieRec(BinaryComparable[], int, int, byte[], int, CarriedTrieNodeRef) private TrieNode buildTrieRec(BinaryComparable[] splits, int lower, int upper, byte[] prefix, int maxDepth, CarriedTrieNodeRef ref) final int depth = prefix.length; // We generate leaves for a single split point as well as for // no split points. if (depth >= maxDepth || lower >= upper - 1) { // If we have two consecutive requests for an unsplit trie node, we // can deliver the same one the second time. if (lower == upper && ref.content != null) { return ref.content; } TrieNode result = LeafTrieNodeFactory(depth, splits, lower, upper); ref.content = lower == upper ? result : null; return result; } InnerTrieNode result = new InnerTrieNode(depth); byte[] trial = Arrays.copyOf(prefix, prefix.length + 1); // append an extra byte on to the prefix int currentBound = lower; for (int ch = 0; ch < 0xFF; ++ch) { trial[depth] = (byte) (ch + 1); lower = currentBound; while (currentBound < upper) { if (splits[currentBound].compareTo(trial, 0, trial.length) >= 0) { break; } currentBound += 1; } trial[depth] = (byte) ch; result.child[0xFF & ch] = buildTrieRec(splits, lower, currentBound, trial, maxDepth, ref); } // pick up the rest trial[depth] = (byte) 0xFF; result.child[0xFF] = buildTrieRec(splits, lower, currentBound, trial, maxDepth, ref); return result;
5327 -1009652395apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None TODO: Should the diskFile be deleted? SATD_ADDED addBlock(Block, File) public File addBlock(Block b, File src) throws IOException // First try without creating subdirectories File file = addBlock(b, src, false, false); return (file != null) ? file : addBlock(b, src, true, true);
5326 -1009652396apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None REMIND - mjc - eventually we should have a timeout system in place to clean up block files left by abandoned clients. We should have some timer in place, so that if a blockfile is created but non-valid, and has been idle for >48 hours, we can GC it safely. SATD_ADDED addBlock(Block, File) public File addBlock(Block b, File src) throws IOException // First try without creating subdirectories File file = addBlock(b, src, false, false); return (file != null) ? file : addBlock(b, src, true, true);
5325 -1009652397apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None TODO: check writer? set writer to the current thread SATD_ADDED addBlock(Block, File) public File addBlock(Block b, File src) throws IOException // First try without creating subdirectories File file = addBlock(b, src, false, false); return (file != null) ? file : addBlock(b, src, true, true);
5324 -1009652398apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Find better place? SATD_ADDED addBlock(Block, File) public File addBlock(Block b, File src) throws IOException // First try without creating subdirectories File file = addBlock(b, src, false, false); return (file != null) ? file : addBlock(b, src, true, true);
5323 -1009652399apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None TODO move this up SATD_ADDED addBlock(Block, File) public File addBlock(Block b, File src) throws IOException // First try without creating subdirectories File file = addBlock(b, src, false, false); return (file != null) ? file : addBlock(b, src, true, true);
5322 -1009652400apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None TODO:FEDERATION valid synchronization SATD_ADDED addBlock(Block, File) public File addBlock(Block b, File src) throws IOException // First try without creating subdirectories File file = addBlock(b, src, false, false); return (file != null) ? file : addBlock(b, src, true, true);
5321 -1009652401apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None TODO valid synchronization SATD_ADDED addBlock(Block, File) public File addBlock(Block b, File src) throws IOException // First try without creating subdirectories File file = addBlock(b, src, false, false); return (file != null) ? file : addBlock(b, src, true, true);
5320 -1009652402apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Files that were being written when the datanode was last shutdown are now moved back to the data directory. It is possible that in the future, we might want to do some sort of datanode-local recovery for these blocks. For example, crc validation. SATD_ADDED addBlock(Block, File) public File addBlock(Block b, File src) throws IOException // First try without creating subdirectories File file = addBlock(b, src, false, false); return (file != null) ? file : addBlock(b, src, true, true);
5319 -1009652403apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None TODO:FEDERATION scalability issue - a thread per DU is needed SATD_ADDED addBlock(Block, File) public File addBlock(Block b, File src) throws IOException // First try without creating subdirectories File file = addBlock(b, src, false, false); return (file != null) ? file : addBlock(b, src, true, true);
5318 -1009652404apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None for security authorization server principal for this call should be JT's one. SATD_ADDED refreshUserToGroupsMappings() private int refreshUserToGroupsMappings() throws IOException // Get the current configuration Configuration conf = getConf(); // for security authorization // server principal for this call // should be JT's one. JobConf jConf = new JobConf(conf); conf.set(CommonConfigurationKeys.HADOOP_SECURITY_SERVICE_USER_NAME_KEY, jConf.get(JobTracker.JT_USER_NAME, "")); // Create the client RefreshUserMappingsProtocol refreshProtocol = (RefreshUserMappingsProtocol) RPC.getProxy(RefreshUserMappingsProtocol.class, RefreshUserMappingsProtocol.versionID, JobTracker.getAddress(conf), getUGI(conf), conf, NetUtils.getSocketFactory(conf, RefreshUserMappingsProtocol.class)); // Refresh the user-to-groups mappings refreshProtocol.refreshUserToGroupsMappings(); return 0;
5317 -1009652405apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None for security authorization server principal for this call should be JT's one. SATD_ADDED refreshSuperUserGroupsConfiguration() public int refreshSuperUserGroupsConfiguration() throws IOException // Get the current configuration Configuration conf = getConf(); // for security authorization // server principal for this call // should be JT's one. JobConf jConf = new JobConf(conf); conf.set(CommonConfigurationKeys.HADOOP_SECURITY_SERVICE_USER_NAME_KEY, jConf.get(JobTracker.JT_USER_NAME, "")); // Create the client RefreshUserMappingsProtocol refreshProtocol = (RefreshUserMappingsProtocol) RPC.getProxy(RefreshUserMappingsProtocol.class, RefreshUserMappingsProtocol.versionID, JobTracker.getAddress(conf), getUGI(conf), conf, NetUtils.getSocketFactory(conf, RefreshUserMappingsProtocol.class)); // Refresh the user-to-groups mappings refreshProtocol.refreshSuperUserGroupsConfiguration(); return 0;
5316 -1009652406apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Refresh the authorization policy in-effect SATD_ADDED refreshAuthorizationPolicy() private int refreshAuthorizationPolicy() throws IOException // Get the current configuration Configuration conf = getConf(); // for security authorization // server principal for this call // should be JT's one. JobConf jConf = new JobConf(conf); conf.set(CommonConfigurationKeys.HADOOP_SECURITY_SERVICE_USER_NAME_KEY, jConf.get(JobTracker.JT_USER_NAME, "")); // Create the client RefreshAuthorizationPolicyProtocol refreshProtocol = (RefreshAuthorizationPolicyProtocol) RPC.getProxy(RefreshAuthorizationPolicyProtocol.class, RefreshAuthorizationPolicyProtocol.versionID, JobTracker.getAddress(conf), getUGI(conf), conf, NetUtils.getSocketFactory(conf, RefreshAuthorizationPolicyProtocol.class)); // Refresh the authorization policy in-effect refreshProtocol.refreshServiceAcl(); return 0;
5315 -1009652407apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None for security authorization server principal for this call should be JT's one. SATD_ADDED refreshAuthorizationPolicy() private int refreshAuthorizationPolicy() throws IOException // Get the current configuration Configuration conf = getConf(); // for security authorization // server principal for this call // should be JT's one. JobConf jConf = new JobConf(conf); conf.set(CommonConfigurationKeys.HADOOP_SECURITY_SERVICE_USER_NAME_KEY, jConf.get(JobTracker.JT_USER_NAME, "")); // Create the client RefreshAuthorizationPolicyProtocol refreshProtocol = (RefreshAuthorizationPolicyProtocol) RPC.getProxy(RefreshAuthorizationPolicyProtocol.class, RefreshAuthorizationPolicyProtocol.versionID, JobTracker.getAddress(conf), getUGI(conf), conf, NetUtils.getSocketFactory(conf, RefreshAuthorizationPolicyProtocol.class)); // Refresh the authorization policy in-effect refreshProtocol.refreshServiceAcl(); return 0;
5314 -1009652408apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Set service-level authorization security policy SATD_ADDED getAsyncDiskService() return asyncDiskService; MRAsyncDiskService getAsyncDiskService()
5313 -1009652409apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None if there are more numSplits than params we're going to introduce some limit and offsets TODO: write code to generate the start/end pairs for each group SATD_ADDED getSplits(JobContext) public static List getSplits(JobContext context) throws IOException Configuration conf = context.getConfiguration(); int numSplits = conf.getInt("mapreduce.job.maps", 1); LOG.debug("creating splits up to " + numSplits); List splits = new ArrayList(); int i = 0; long start = 0; long end = 0; boolean limitOffset = true; // This is the fancy part of mapping inputs...here's how we figure out // splits // get the params query or the params VerticaConfiguration config = new VerticaConfiguration(conf); String inputQuery = config.getInputQuery(); if (inputQuery == null) throw new IOException("Vertica input requires query defined by " + VerticaConfiguration.QUERY_PROP); String paramsQuery = config.getParamsQuery(); Collection> params = config.getInputParameters(); // TODO: limit needs order by unique key // TODO: what if there are more parameters than numsplits? // prep a count(*) wrapper query and then populate the bind params for each String countQuery = "SELECT COUNT(*) FROM (\n" + inputQuery + "\n) count"; if (paramsQuery != null) { LOG.debug("creating splits using paramsQuery :" + paramsQuery); Connection conn = null; Statement stmt = null; try { conn = config.getConnection(false); stmt = conn.createStatement(); ResultSet rs = stmt.executeQuery(paramsQuery); ResultSetMetaData rsmd = rs.getMetaData(); while (rs.next()) { limitOffset = false; List segmentParams = new ArrayList(); for (int j = 1; j <= rsmd.getColumnCount(); j++) { segmentParams.add(rs.getObject(j)); } splits.add(new VerticaInputSplit(inputQuery, segmentParams, start, end)); } } catch (Exception e) { throw new IOException(e); } finally { try { if (stmt != null) stmt.close(); } catch (SQLException e) { throw new IOException(e); } } } else if (params != null && params.size() > 0) { LOG.debug("creating splits using " + params.size() + " params"); limitOffset = false; for (List segmentParams : params) { // if there are more numSplits than params we're going to introduce some // limit and offsets // TODO: write code to generate the start/end pairs for each group splits.add(new VerticaInputSplit(inputQuery, segmentParams, start, end)); } } if (limitOffset) { LOG.debug("creating splits using limit and offset"); Connection conn = null; Statement stmt = null; long count = 0; try { conn = config.getConnection(false); stmt = conn.createStatement(); ResultSet rs = stmt.executeQuery(countQuery); rs.next(); count = rs.getLong(1); } catch (Exception e) { throw new IOException(e); } finally { try { if (stmt != null) stmt.close(); } catch (SQLException e) { throw new IOException(e); } } long splitSize = count / numSplits; end = splitSize; LOG.debug("creating " + numSplits + " splits for " + count + " records"); for (i = 0; i < numSplits; i++) { splits.add(new VerticaInputSplit(inputQuery, null, start, end)); start += splitSize; end += splitSize; } } LOG.debug("returning " + splits.size() + " final splits"); return splits;
5312 -1009652410apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None TODO: limit needs order by unique key TODO: what if there are more parameters than numsplits? prep a count(*) wrapper query and then populate the bind params for each SATD_ADDED getSplits(JobContext) public static List getSplits(JobContext context) throws IOException Configuration conf = context.getConfiguration(); int numSplits = conf.getInt("mapreduce.job.maps", 1); LOG.debug("creating splits up to " + numSplits); List splits = new ArrayList(); int i = 0; long start = 0; long end = 0; boolean limitOffset = true; // This is the fancy part of mapping inputs...here's how we figure out // splits // get the params query or the params VerticaConfiguration config = new VerticaConfiguration(conf); String inputQuery = config.getInputQuery(); if (inputQuery == null) throw new IOException("Vertica input requires query defined by " + VerticaConfiguration.QUERY_PROP); String paramsQuery = config.getParamsQuery(); Collection> params = config.getInputParameters(); // TODO: limit needs order by unique key // TODO: what if there are more parameters than numsplits? // prep a count(*) wrapper query and then populate the bind params for each String countQuery = "SELECT COUNT(*) FROM (\n" + inputQuery + "\n) count"; if (paramsQuery != null) { LOG.debug("creating splits using paramsQuery :" + paramsQuery); Connection conn = null; Statement stmt = null; try { conn = config.getConnection(false); stmt = conn.createStatement(); ResultSet rs = stmt.executeQuery(paramsQuery); ResultSetMetaData rsmd = rs.getMetaData(); while (rs.next()) { limitOffset = false; List segmentParams = new ArrayList(); for (int j = 1; j <= rsmd.getColumnCount(); j++) { segmentParams.add(rs.getObject(j)); } splits.add(new VerticaInputSplit(inputQuery, segmentParams, start, end)); } } catch (Exception e) { throw new IOException(e); } finally { try { if (stmt != null) stmt.close(); } catch (SQLException e) { throw new IOException(e); } } } else if (params != null && params.size() > 0) { LOG.debug("creating splits using " + params.size() + " params"); limitOffset = false; for (List segmentParams : params) { // if there are more numSplits than params we're going to introduce some // limit and offsets // TODO: write code to generate the start/end pairs for each group splits.add(new VerticaInputSplit(inputQuery, segmentParams, start, end)); } } if (limitOffset) { LOG.debug("creating splits using limit and offset"); Connection conn = null; Statement stmt = null; long count = 0; try { conn = config.getConnection(false); stmt = conn.createStatement(); ResultSet rs = stmt.executeQuery(countQuery); rs.next(); count = rs.getLong(1); } catch (Exception e) { throw new IOException(e); } finally { try { if (stmt != null) stmt.close(); } catch (SQLException e) { throw new IOException(e); } } long splitSize = count / numSplits; end = splitSize; LOG.debug("creating " + numSplits + " splits for " + count + " records"); for (i = 0; i < numSplits; i++) { splits.add(new VerticaInputSplit(inputQuery, null, start, end)); start += splitSize; end += splitSize; } } LOG.debug("returning " + splits.size() + " final splits"); return splits;
5311 -1009652411apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None This is the fancy part of mapping inputs...here's how we figure out splits get the params query or the params SATD_ADDED getSplits(JobContext) public static List getSplits(JobContext context) throws IOException Configuration conf = context.getConfiguration(); int numSplits = conf.getInt("mapreduce.job.maps", 1); LOG.debug("creating splits up to " + numSplits); List splits = new ArrayList(); int i = 0; long start = 0; long end = 0; boolean limitOffset = true; // This is the fancy part of mapping inputs...here's how we figure out // splits // get the params query or the params VerticaConfiguration config = new VerticaConfiguration(conf); String inputQuery = config.getInputQuery(); if (inputQuery == null) throw new IOException("Vertica input requires query defined by " + VerticaConfiguration.QUERY_PROP); String paramsQuery = config.getParamsQuery(); Collection> params = config.getInputParameters(); // TODO: limit needs order by unique key // TODO: what if there are more parameters than numsplits? // prep a count(*) wrapper query and then populate the bind params for each String countQuery = "SELECT COUNT(*) FROM (\n" + inputQuery + "\n) count"; if (paramsQuery != null) { LOG.debug("creating splits using paramsQuery :" + paramsQuery); Connection conn = null; Statement stmt = null; try { conn = config.getConnection(false); stmt = conn.createStatement(); ResultSet rs = stmt.executeQuery(paramsQuery); ResultSetMetaData rsmd = rs.getMetaData(); while (rs.next()) { limitOffset = false; List segmentParams = new ArrayList(); for (int j = 1; j <= rsmd.getColumnCount(); j++) { segmentParams.add(rs.getObject(j)); } splits.add(new VerticaInputSplit(inputQuery, segmentParams, start, end)); } } catch (Exception e) { throw new IOException(e); } finally { try { if (stmt != null) stmt.close(); } catch (SQLException e) { throw new IOException(e); } } } else if (params != null && params.size() > 0) { LOG.debug("creating splits using " + params.size() + " params"); limitOffset = false; for (List segmentParams : params) { // if there are more numSplits than params we're going to introduce some // limit and offsets // TODO: write code to generate the start/end pairs for each group splits.add(new VerticaInputSplit(inputQuery, segmentParams, start, end)); } } if (limitOffset) { LOG.debug("creating splits using limit and offset"); Connection conn = null; Statement stmt = null; long count = 0; try { conn = config.getConnection(false); stmt = conn.createStatement(); ResultSet rs = stmt.executeQuery(countQuery); rs.next(); count = rs.getLong(1); } catch (Exception e) { throw new IOException(e); } finally { try { if (stmt != null) stmt.close(); } catch (SQLException e) { throw new IOException(e); } } long splitSize = count / numSplits; end = splitSize; LOG.debug("creating " + numSplits + " splits for " + count + " records"); for (i = 0; i < numSplits; i++) { splits.add(new VerticaInputSplit(inputQuery, null, start, end)); start += splitSize; end += splitSize; } } LOG.debug("returning " + splits.size() + " final splits"); return splits;
5310 -1009652412apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None TODO: catch when params required but missing TODO: better error message when count query is bad SATD_ADDED getSplits(JobContext) public static List getSplits(JobContext context) throws IOException Configuration conf = context.getConfiguration(); int numSplits = conf.getInt("mapreduce.job.maps", 1); LOG.debug("creating splits up to " + numSplits); List splits = new ArrayList(); int i = 0; long start = 0; long end = 0; boolean limitOffset = true; // This is the fancy part of mapping inputs...here's how we figure out // splits // get the params query or the params VerticaConfiguration config = new VerticaConfiguration(conf); String inputQuery = config.getInputQuery(); if (inputQuery == null) throw new IOException("Vertica input requires query defined by " + VerticaConfiguration.QUERY_PROP); String paramsQuery = config.getParamsQuery(); Collection> params = config.getInputParameters(); // TODO: limit needs order by unique key // TODO: what if there are more parameters than numsplits? // prep a count(*) wrapper query and then populate the bind params for each String countQuery = "SELECT COUNT(*) FROM (\n" + inputQuery + "\n) count"; if (paramsQuery != null) { LOG.debug("creating splits using paramsQuery :" + paramsQuery); Connection conn = null; Statement stmt = null; try { conn = config.getConnection(false); stmt = conn.createStatement(); ResultSet rs = stmt.executeQuery(paramsQuery); ResultSetMetaData rsmd = rs.getMetaData(); while (rs.next()) { limitOffset = false; List segmentParams = new ArrayList(); for (int j = 1; j <= rsmd.getColumnCount(); j++) { segmentParams.add(rs.getObject(j)); } splits.add(new VerticaInputSplit(inputQuery, segmentParams, start, end)); } } catch (Exception e) { throw new IOException(e); } finally { try { if (stmt != null) stmt.close(); } catch (SQLException e) { throw new IOException(e); } } } else if (params != null && params.size() > 0) { LOG.debug("creating splits using " + params.size() + " params"); limitOffset = false; for (List segmentParams : params) { // if there are more numSplits than params we're going to introduce some // limit and offsets // TODO: write code to generate the start/end pairs for each group splits.add(new VerticaInputSplit(inputQuery, segmentParams, start, end)); } } if (limitOffset) { LOG.debug("creating splits using limit and offset"); Connection conn = null; Statement stmt = null; long count = 0; try { conn = config.getConnection(false); stmt = conn.createStatement(); ResultSet rs = stmt.executeQuery(countQuery); rs.next(); count = rs.getLong(1); } catch (Exception e) { throw new IOException(e); } finally { try { if (stmt != null) stmt.close(); } catch (SQLException e) { throw new IOException(e); } } long splitSize = count / numSplits; end = splitSize; LOG.debug("creating " + numSplits + " splits for " + count + " records"); for (i = 0; i < numSplits; i++) { splits.add(new VerticaInputSplit(inputQuery, null, start, end)); start += splitSize; end += splitSize; } } LOG.debug("returning " + splits.size() + " final splits"); return splits;
5309 -1009652413apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None TODO: create segmented projections SATD_ADDED checkOutputSpecs(Configuration) public static void checkOutputSpecs(Configuration conf) throws IOException VerticaConfiguration vtconfig = new VerticaConfiguration(conf); String writerTable = vtconfig.getOutputTableName(); if (writerTable == null) throw new IOException("Vertica output requires a table name defined by " + VerticaConfiguration.OUTPUT_TABLE_NAME_PROP); String[] def = vtconfig.getOutputTableDef(); boolean dropTable = vtconfig.getDropTable(); String schema = null; String table = null; String[] schemaTable = writerTable.split("\\."); if (schemaTable.length == 2) { schema = schemaTable[0]; table = schemaTable[1]; } else table = schemaTable[0]; Statement stmt = null; try { Connection conn = vtconfig.getConnection(true); DatabaseMetaData dbmd = conn.getMetaData(); ResultSet rs = dbmd.getTables(null, schema, table, null); boolean tableExists = rs.next(); stmt = conn.createStatement(); if (tableExists && dropTable) { if (verticaVersion(conf, true) >= 305) { stmt = conn.createStatement(); stmt.execute("TRUNCATE TABLE " + writerTable); } else { // for version < 3.0 drop the table if it exists // if def is empty, grab the columns first to redfine the table if (def == null) { rs = dbmd.getColumns(null, schema, table, null); ArrayList defs = new ArrayList(); while (rs.next()) defs.add(rs.getString(4) + " " + rs.getString(5)); def = defs.toArray(new String[0]); } stmt = conn.createStatement(); stmt.execute("DROP TABLE " + writerTable + " CASCADE"); // force create tableExists = false; } } // create table if it doesn't exist if (!tableExists) { if (def == null) throw new RuntimeException("Table " + writerTable + " does not exist and no table definition provided"); if (schema != null) { rs = dbmd.getSchemas(null, schema); if (!rs.next()) stmt.execute("CREATE SCHEMA " + schema); } StringBuffer tabledef = new StringBuffer("CREATE TABLE ").append(writerTable).append(" ("); for (String column : def) tabledef.append(column).append(","); tabledef.replace(tabledef.length() - 1, tabledef.length(), ")"); stmt.execute(tabledef.toString()); // TODO: create segmented projections stmt.execute("select implement_temp_design('" + writerTable + "')"); } } catch (Exception e) { throw new RuntimeException(e); } finally { if (stmt != null) try { stmt.close(); } catch (SQLException e) { throw new RuntimeException(e); } }
5308 -1009652414apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None spawn the JVM in a new thread. Note that there will be very little extra overhead of launching the new thread for a new JVM since most of the cost is involved in launching the process. Moreover, since we are going to be using the JVM for running many tasks, the thread launch cost becomes trivial when amortized over all tasks. Doing it this way also keeps code simple. SATD_ADDED constructJvmEnv(List, Vector, File, File, long, File, Map, JobConf) public JvmEnv constructJvmEnv(List setup, Vector vargs, File stdout, File stderr, long logSize, File workDir, Map env, JobConf conf) return new JvmEnv(setup, vargs, stdout, stderr, logSize, workDir, env, conf);
5307 -1009652415apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None TODO: move it to DatanodeID once DatanodeID is not stored in FSImage SATD_ADDED readFields(DataInput) public void readFields(DataInput in) throws IOException super.readFields(in); // TODO: move it to DatanodeID once DatanodeID is not stored in FSImage this.ipcPort = in.readShort() & 0x0000ffff; this.capacity = in.readLong(); this.dfsUsed = in.readLong(); this.remaining = in.readLong(); this.blockPoolUsed = in.readLong(); this.lastUpdate = in.readLong(); this.xceiverCount = in.readInt(); this.location = Text.readString(in); this.hostName = Text.readString(in); setAdminState(WritableUtils.readEnum(in, AdminStates.class));
5306 -1009652416apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None TODO: move it to DatanodeID once DatanodeID is not stored in FSImage SATD_ADDED write(DataOutput) public void write(DataOutput out) throws IOException super.write(out); // TODO: move it to DatanodeID once DatanodeID is not stored in FSImage out.writeShort(ipcPort); out.writeLong(capacity); out.writeLong(dfsUsed); out.writeLong(remaining); out.writeLong(blockPoolUsed); out.writeLong(lastUpdate); out.writeInt(xceiverCount); Text.writeString(out, location); Text.writeString(out, hostName == null ? "" : hostName); WritableUtils.writeEnum(out, getAdminState());
5305 -1009652417apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None This simply marks the job as completed. Note that the caller is responsible for raising events. SATD_ADDED getJobTokenSecretManager() JobTokenSecretManager getJobTokenSecretManager() return jobTokenSecretManager;
5304 -1009652418apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Set service-level authorization security policy SATD_ADDED getJobTokenSecretManager() return jobTokenSecretManager; JobTokenSecretManager getJobTokenSecretManager()
5303 -1009652419apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Write back the new restart count and rename the old info file TODO This is similar to jobhistory recovery, maybe this common code can be factored out. SATD_ADDED getJobTokenSecretManager() return jobTokenSecretManager; JobTokenSecretManager getJobTokenSecretManager()
5302 -1009652420apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Simple check to make sure that capacity is properly distributed among its children. level 1 children SATD_ADDED testMinCapacity() public void testMinCapacity() AbstractQueue rt = QueueHierarchyBuilder.createRootAbstractQueue(); // Simple check to make sure that capacity is properly distributed among // its children. // level 1 children QueueSchedulingContext a1 = new QueueSchedulingContext("a", 25, -1, -1); QueueSchedulingContext a2 = new QueueSchedulingContext("b", 25, -1, -1); AbstractQueue q = new ContainerQueue(rt, a1); AbstractQueue ql = new ContainerQueue(rt, a2); // level 2 children QueueSchedulingContext a = new QueueSchedulingContext("aa", 50, -1, -1); QueueSchedulingContext b = new QueueSchedulingContext("ab", 50, -1, -1); QueueSchedulingContext c = new QueueSchedulingContext("ac", 50, -1, -1); QueueSchedulingContext d = new QueueSchedulingContext("ad", 50, -1, -1); AbstractQueue q1 = new JobQueue(q, a); AbstractQueue q2 = new JobQueue(q, b); AbstractQueue q3 = new JobQueue(ql, c); AbstractQueue q4 = new JobQueue(ql, d); rt.update(1000, 1000); // share at level 0. // (1000 * 25) / 100 assertEquals(q.getQueueSchedulingContext().getMapTSC().getCapacity(), 250); assertEquals(ql.getQueueSchedulingContext().getMapTSC().getCapacity(), 250); // share would be (1000 * 25 / 100 ) * (50 / 100) assertEquals(q1.getQueueSchedulingContext().getMapTSC().getCapacity(), 125); assertEquals(q2.getQueueSchedulingContext().getMapTSC().getCapacity(), 125); assertEquals(q3.getQueueSchedulingContext().getMapTSC().getCapacity(), 125); assertEquals(q4.getQueueSchedulingContext().getMapTSC().getCapacity(), 125); // rt.update(1, 1); assertEquals(q.getQueueSchedulingContext().getMapTSC().getCapacity(), 0); assertEquals(ql.getQueueSchedulingContext().getMapTSC().getCapacity(), 0); // share would be (1000 * 25 / 100 ) * (50 / 100) assertEquals(q1.getQueueSchedulingContext().getMapTSC().getCapacity(), 0); assertEquals(q2.getQueueSchedulingContext().getMapTSC().getCapacity(), 0); assertEquals(q3.getQueueSchedulingContext().getMapTSC().getCapacity(), 0); assertEquals(q4.getQueueSchedulingContext().getMapTSC().getCapacity(), 0);
5301 -1009652421apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None XXX don't know what the file is? SATD_ADDED upload(IProgressMonitor, File) public void upload(IProgressMonitor monitor, final File file) throws IOException if (file.isDirectory()) { Path filePath = new Path(this.path, file.getName()); getDFS().mkdirs(filePath); DFSFolder newFolder = new DFSFolder(this, filePath); monitor.worked(1); for (File child : file.listFiles()) { if (monitor.isCanceled()) return; newFolder.upload(monitor, child); } } else if (file.isFile()) { Path filePath = new Path(this.path, file.getName()); DFSFile newFile = new DFSFile(this, filePath, file, monitor); } else { // XXX don't know what the file is? }
5300 -1009652422apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Hack for local FS that does not have the concept of a 'mounting point' SATD_ADDED testWithoutCounters() public void testWithoutCounters() throws Exception _testMultipleOutputs(false); _testMOWithJavaSerialization(false);
5299 -1009652423apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Hack for local FS that does not have the concept of a 'mounting point' SATD_ADDED testOKRun() public void testOKRun() throws Exception run(false, false);
5298 -1009652424apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None The appender should be able to close properly SATD_ADDED setUp() public void setUp() throws Exception this.conf = new Configuration(); if (simulatedStorage) { conf.setBoolean(SimulatedFSDataset.CONFIG_PROPERTY_SIMULATED, true); } conf.setBoolean(DFSConfigKeys.DFS_SUPPORT_APPEND_KEY, true); // lower heartbeat interval for fast recognition of DN death conf.setInt(DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, 1000); conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1); conf.setInt(DFSConfigKeys.DFS_CLIENT_SOCKET_TIMEOUT_KEY, 5000); // handle under-replicated blocks quickly (for replication asserts) conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_PENDING_TIMEOUT_SEC_KEY, 5); conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1); // handle failures in the DFSClient pipeline quickly // (for cluster.shutdown(); fs.close() idiom) conf.setInt("ipc.client.connect.max.retries", 1);
5297 -1009652425apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None This really shouldn't happen... SATD_ADDED isValidRequestor(String, Configuration) protected boolean isValidRequestor(String remoteUser, Configuration conf) throws IOException if (remoteUser == null) { // This really shouldn't happen... LOG.warn("Received null remoteUser while authorizing access to getImage servlet"); return false; } String[] validRequestors = { SecurityUtil.getServerPrincipal(conf.get(DFSConfigKeys.DFS_NAMENODE_KRB_HTTPS_USER_NAME_KEY), NameNode.getAddress(conf).getHostName()), SecurityUtil.getServerPrincipal(conf.get(DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY), NameNode.getAddress(conf).getHostName()), SecurityUtil.getServerPrincipal(conf.get(DFSConfigKeys.DFS_SECONDARY_NAMENODE_KRB_HTTPS_USER_NAME_KEY), SecondaryNameNode.getHttpAddress(conf).getHostName()), SecurityUtil.getServerPrincipal(conf.get(DFSConfigKeys.DFS_SECONDARY_NAMENODE_USER_NAME_KEY), SecondaryNameNode.getHttpAddress(conf).getHostName()) }; for (String v : validRequestors) { if (v != null && v.equals(remoteUser)) { if (LOG.isDebugEnabled()) LOG.debug("isValidRequestor is allowing: " + remoteUser); return true; } } if (LOG.isDebugEnabled()) LOG.debug("isValidRequestor is rejecting: " + remoteUser); return false;
5296 -1009652426apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None can only happen because of a software bug. the bug should be fixed. SATD_ADDED updateCountForINodeWithQuota(INodeDirectory, INode.DirCounts, ArrayList) private static void updateCountForINodeWithQuota(INodeDirectory dir, INode.DirCounts counts, ArrayList nodesInPath) long parentNamespace = counts.nsCount; long parentDiskspace = counts.dsCount; // for self. should not call node.spaceConsumedInTree() counts.nsCount = 1L; counts.dsCount = 0L; /* We don't need nodesInPath if we could use 'parent' field in * INode. using 'parent' is not currently recommended. */ nodesInPath.add(dir); for (INode child : dir.getChildren()) { if (child.isDirectory()) { updateCountForINodeWithQuota((INodeDirectory) child, counts, nodesInPath); } else if (child.isLink()) { counts.nsCount += 1; } else { // reduce recursive calls counts.nsCount += 1; counts.dsCount += ((INodeFile) child).diskspaceConsumed(); } } if (dir.isQuotaSet()) { ((INodeDirectoryWithQuota) dir).setSpaceConsumed(counts.nsCount, counts.dsCount); // check if quota is violated for some reason. if ((dir.getNsQuota() >= 0 && counts.nsCount > dir.getNsQuota()) || (dir.getDsQuota() >= 0 && counts.dsCount > dir.getDsQuota())) { // can only happen because of a software bug. the bug should be fixed. StringBuilder path = new StringBuilder(512); for (INode n : nodesInPath) { path.append('/'); path.append(n.getLocalName()); } NameNode.LOG.warn("Quota violation in image for " + path + " (Namespace quota : " + dir.getNsQuota() + " consumed : " + counts.nsCount + ")" + " (Diskspace quota : " + dir.getDsQuota() + " consumed : " + counts.dsCount + ")."); } } // pop nodesInPath.remove(nodesInPath.size() - 1); counts.nsCount += parentNamespace; counts.dsCount += parentDiskspace;
5295 -1009652427apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None do the move SATD_ADDED unprotectedConcat(String, String[], long) public void unprotectedConcat(String target, String[] srcs, long timestamp) throws UnresolvedLinkException assert hasWriteLock(); if (NameNode.stateChangeLog.isDebugEnabled()) { NameNode.stateChangeLog.debug("DIR* FSNamesystem.concat to " + target); } // do the move INode[] trgINodes = getExistingPathINodes(target); INodeFile trgInode = (INodeFile) trgINodes[trgINodes.length - 1]; INodeDirectory trgParent = (INodeDirectory) trgINodes[trgINodes.length - 2]; INodeFile[] allSrcInodes = new INodeFile[srcs.length]; int i = 0; int totalBlocks = 0; for (String src : srcs) { INodeFile srcInode = getFileINode(src); allSrcInodes[i++] = srcInode; totalBlocks += srcInode.blocks.length; } // copy the blocks trgInode.appendBlocks(allSrcInodes, totalBlocks); // since we are in the same dir - we can use same parent to remove files int count = 0; for (INodeFile nodeToRemove : allSrcInodes) { if (nodeToRemove == null) continue; nodeToRemove.blocks = null; trgParent.removeChild(nodeToRemove); count++; } trgInode.setModificationTimeForce(timestamp); trgParent.setModificationTime(timestamp); // update quota on the parent directory ('count' files removed, 0 space) unprotectedUpdateCount(trgINodes, trgINodes.length - 1, -count, 0);
5293 -1009652429apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Check the permissions of the task-controller binary by running it plainly. If permissions are correct, it returns an error code 1, else it returns 24 or something else if some other bugs are also present. SATD_ADDED setup() public void setup() throws IOException super.setup(); // Check the permissions of the task-controller binary by running it plainly. // If permissions are correct, it returns an error code 1, else it returns // 24 or something else if some other bugs are also present. String[] taskControllerCmd = new String[] { getTaskControllerExecutablePath() }; ShellCommandExecutor shExec = new ShellCommandExecutor(taskControllerCmd); try { shExec.execute(); } catch (ExitCodeException e) { int exitCode = shExec.getExitCode(); if (exitCode != 1) { LOG.warn("Exit code from checking binary permissions is : " + exitCode); logOutput(shExec.getOutput()); throw new IOException("Task controller setup failed because of invalid" + "permissions/ownership with exit code " + exitCode, e); } }
5292 -1009652430apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Split the special file that contains the list of distributed cache file paths and their file sizes such that each split corresponds to approximately same amount of distributed cache data to be generated. Consider numTaskTrackers * numMapSlotsPerTracker as the number of maps for this job, if there is lot of data to be generated. SATD_ADDED getSplits(JobContext) public List getSplits(JobContext jobCtxt) throws IOException final JobConf jobConf = new JobConf(jobCtxt.getConfiguration()); final JobClient client = new JobClient(jobConf); ClusterStatus stat = client.getClusterStatus(true); int numTrackers = stat.getTaskTrackers(); final int fileCount = jobConf.getInt(GRIDMIX_DISTCACHE_FILE_COUNT, -1); // Total size of distributed cache files to be generated final long totalSize = jobConf.getLong(GRIDMIX_DISTCACHE_BYTE_COUNT, -1); // Get the path of the special file String distCacheFileList = jobConf.get(GRIDMIX_DISTCACHE_FILE_LIST); if (fileCount < 0 || totalSize < 0 || distCacheFileList == null) { throw new RuntimeException("Invalid metadata: #files (" + fileCount + "), total_size (" + totalSize + "), filelisturi (" + distCacheFileList + ")"); } Path sequenceFile = new Path(distCacheFileList); FileSystem fs = sequenceFile.getFileSystem(jobConf); FileStatus srcst = fs.getFileStatus(sequenceFile); // Consider the number of TTs * mapSlotsPerTracker as number of mappers. int numMapSlotsPerTracker = jobConf.getInt(TTConfig.TT_MAP_SLOTS, 2); int numSplits = numTrackers * numMapSlotsPerTracker; List splits = new ArrayList(numSplits); LongWritable key = new LongWritable(); BytesWritable value = new BytesWritable(); // Average size of data to be generated by each map task final long targetSize = Math.max(totalSize / numSplits, DistributedCacheEmulator.AVG_BYTES_PER_MAP); long splitStartPosition = 0L; long splitEndPosition = 0L; long acc = 0L; long bytesRemaining = srcst.getLen(); SequenceFile.Reader reader = null; try { reader = new SequenceFile.Reader(fs, sequenceFile, jobConf); while (reader.next(key, value)) { // If adding this file would put this split past the target size, // cut the last split and put this file in the next split. if (acc + key.get() > targetSize && acc != 0) { long splitSize = splitEndPosition - splitStartPosition; splits.add(new FileSplit(sequenceFile, splitStartPosition, splitSize, (String[]) null)); bytesRemaining -= splitSize; splitStartPosition = splitEndPosition; acc = 0L; } acc += key.get(); splitEndPosition = reader.getPosition(); } } finally { if (reader != null) { reader.close(); } } if (bytesRemaining != 0) { splits.add(new FileSplit(sequenceFile, splitStartPosition, bytesRemaining, (String[]) null)); } return splits;
5291 -1009652431apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None XXX There may be redundant location info available in the event. We might consider extracting it from this event. Currently this is redundant, but making this will add future-proofing. SATD_ADDED processMapAttemptFinishedEvent(MapAttemptFinishedEvent) private void processMapAttemptFinishedEvent(MapAttemptFinishedEvent event) LoggedTaskAttempt attempt = getOrMakeTaskAttempt(event.getTaskType(), event.getTaskId().toString(), event.getAttemptId().toString()); if (attempt == null) { return; } attempt.setResult(getPre21Value(event.getTaskStatus())); attempt.setHostName(event.getHostname()); // XXX There may be redundant location info available in the event. // We might consider extracting it from this event. Currently this // is redundant, but making this will add future-proofing. attempt.setFinishTime(event.getFinishTime()); attempt.incorporateCounters(((MapAttemptFinished) event.getDatum()).counters);
5290 -1009652432apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None XXX There may be redundant location info available in the event. We might consider extracting it from this event. Currently this is redundant, but making this will add future-proofing. SATD_ADDED processReduceAttemptFinishedEvent(ReduceAttemptFinishedEvent) private void processReduceAttemptFinishedEvent(ReduceAttemptFinishedEvent event) LoggedTaskAttempt attempt = getOrMakeTaskAttempt(event.getTaskType(), event.getTaskId().toString(), event.getAttemptId().toString()); if (attempt == null) { return; } attempt.setResult(getPre21Value(event.getTaskStatus())); attempt.setHostName(event.getHostname()); // XXX There may be redundant location info available in the event. // We might consider extracting it from this event. Currently this // is redundant, but making this will add future-proofing. attempt.setFinishTime(event.getFinishTime()); attempt.setShuffleFinished(event.getShuffleFinishTime()); attempt.setSortFinished(event.getSortFinishTime()); attempt.incorporateCounters(((ReduceAttemptFinished) event.getDatum()).counters);
5289 -1009652433apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None TODO remove this once the deprecate APIs in LoggedJob are removed SATD_ADDED process(Properties) public void process(Properties conf) if (finalized) { throw new IllegalStateException("JobBuilder.process(Properties conf) called after LoggedJob built"); } // TODO remove this once the deprecate APIs in LoggedJob are removed result.setQueue(extract(conf, JobConfPropertyNames.QUEUE_NAMES.getCandidates(), "default")); result.setJobName(extract(conf, JobConfPropertyNames.JOB_NAMES.getCandidates(), null)); maybeSetHeapMegabytes(extractMegabytes(conf, JobConfPropertyNames.TASK_JAVA_OPTS_S.getCandidates())); maybeSetJobMapMB(extractMegabytes(conf, JobConfPropertyNames.MAP_JAVA_OPTS_S.getCandidates())); maybeSetJobReduceMB(extractMegabytes(conf, JobConfPropertyNames.REDUCE_JAVA_OPTS_S.getCandidates())); this.jobConfigurationParameters = conf;
5288 -1009652434apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Now we need to generate the random numbers according to the above distribution. We create a lot of map tasks, each of which takes at least one \"line\" of the distribution. (That is, a certain number X is to be generated Y number of times.) A map task emits Y key/val pairs. The val is X. The key is a randomly-generated number. The reduce task gets its input sorted by key. That is, sorted in random order. It then emits a single line of text that for the given values. It does not emit the key. Because there's just one reduce task, we emit a single big file of random numbers. SATD_ADDED launch() public void launch() throws Exception // // Generate distribution of ints. This is the answer key. // JobConf conf = null; // Check to get configuration and check if it is configured thro' Configured // interface. This would happen when running testcase thro' command line. if (getConf() == null) { conf = new JobConf(); } else { conf = new JobConf(getConf()); } conf.setJarByClass(TestMapRed.class); int countsToGo = counts; int[] dist = new int[range]; for (int i = 0; i < range; i++) { double avgInts = (1.0 * countsToGo) / (range - i); dist[i] = (int) Math.max(0, Math.round(avgInts + (Math.sqrt(avgInts) * r.nextGaussian()))); countsToGo -= dist[i]; } if (countsToGo > 0) { dist[dist.length - 1] += countsToGo; } // // Write the answer key to a file. // FileSystem fs = FileSystem.get(conf); Path testdir = new Path("mapred.loadtest"); if (!fs.mkdirs(testdir)) { throw new IOException("Mkdirs failed to create " + testdir.toString()); } Path randomIns = new Path(testdir, "genins"); if (!fs.mkdirs(randomIns)) { throw new IOException("Mkdirs failed to create " + randomIns.toString()); } Path answerkey = new Path(randomIns, "answer.key"); SequenceFile.Writer out = SequenceFile.createWriter(fs, conf, answerkey, IntWritable.class, IntWritable.class, SequenceFile.CompressionType.NONE); try { for (int i = 0; i < range; i++) { out.append(new IntWritable(i), new IntWritable(dist[i])); } } finally { out.close(); } // printFiles(randomIns, conf); // // Now we need to generate the random numbers according to // the above distribution. // // We create a lot of map tasks, each of which takes at least // one "line" of the distribution. (That is, a certain number // X is to be generated Y number of times.) // // A map task emits Y key/val pairs. The val is X. The key // is a randomly-generated number. // // The reduce task gets its input sorted by key. That is, sorted // in random order. It then emits a single line of text that // for the given values. It does not emit the key. // // Because there's just one reduce task, we emit a single big // file of random numbers. // Path randomOuts = new Path(testdir, "genouts"); fs.delete(randomOuts, true); JobConf genJob = new JobConf(conf, TestMapRed.class); FileInputFormat.setInputPaths(genJob, randomIns); genJob.setInputFormat(SequenceFileInputFormat.class); genJob.setMapperClass(RandomGenMapper.class); FileOutputFormat.setOutputPath(genJob, randomOuts); genJob.setOutputKeyClass(IntWritable.class); genJob.setOutputValueClass(IntWritable.class); genJob.setOutputFormat(TextOutputFormat.class); genJob.setReducerClass(RandomGenReducer.class); genJob.setNumReduceTasks(1); JobClient.runJob(genJob); // printFiles(randomOuts, conf); // // Next, we read the big file in and regenerate the // original map. It's split into a number of parts. // (That number is 'intermediateReduces'.) // // We have many map tasks, each of which read at least one // of the output numbers. For each number read in, the // map task emits a key/value pair where the key is the // number and the value is "1". // // We have a single reduce task, which receives its input // sorted by the key emitted above. For each key, there will // be a certain number of "1" values. The reduce task sums // these values to compute how many times the given key was // emitted. // // The reduce task then emits a key/val pair where the key // is the number in question, and the value is the number of // times the key was emitted. This is the same format as the // original answer key (except that numbers emitted zero times // will not appear in the regenerated key.) The answer set // is split into a number of pieces. A final MapReduce job // will merge them. // // There's not really a need to go to 10 reduces here // instead of 1. But we want to test what happens when // you have multiple reduces at once. // int intermediateReduces = 10; Path intermediateOuts = new Path(testdir, "intermediateouts"); fs.delete(intermediateOuts, true); JobConf checkJob = new JobConf(conf, TestMapRed.class); FileInputFormat.setInputPaths(checkJob, randomOuts); checkJob.setInputFormat(TextInputFormat.class); checkJob.setMapperClass(RandomCheckMapper.class); FileOutputFormat.setOutputPath(checkJob, intermediateOuts); checkJob.setOutputKeyClass(IntWritable.class); checkJob.setOutputValueClass(IntWritable.class); checkJob.setOutputFormat(MapFileOutputFormat.class); checkJob.setReducerClass(RandomCheckReducer.class); checkJob.setNumReduceTasks(intermediateReduces); JobClient.runJob(checkJob); // printFiles(intermediateOuts, conf); // // OK, now we take the output from the last job and // merge it down to a single file. The map() and reduce() // functions don't really do anything except reemit tuples. // But by having a single reduce task here, we end up merging // all the files. // Path finalOuts = new Path(testdir, "finalouts"); fs.delete(finalOuts, true); JobConf mergeJob = new JobConf(conf, TestMapRed.class); FileInputFormat.setInputPaths(mergeJob, intermediateOuts); mergeJob.setInputFormat(SequenceFileInputFormat.class); mergeJob.setMapperClass(MergeMapper.class); FileOutputFormat.setOutputPath(mergeJob, finalOuts); mergeJob.setOutputKeyClass(IntWritable.class); mergeJob.setOutputValueClass(IntWritable.class); mergeJob.setOutputFormat(SequenceFileOutputFormat.class); mergeJob.setReducerClass(MergeReducer.class); mergeJob.setNumReduceTasks(1); JobClient.runJob(mergeJob); // printFiles(finalOuts, conf); // // Finally, we compare the reconstructed answer key with the // original one. Remember, we need to ignore zero-count items // in the original key. // boolean success = true; Path recomputedkey = new Path(finalOuts, "part-00000"); SequenceFile.Reader in = new SequenceFile.Reader(fs, recomputedkey, conf); int totalseen = 0; try { IntWritable key = new IntWritable(); IntWritable val = new IntWritable(); for (int i = 0; i < range; i++) { if (dist[i] == 0) { continue; } if (!in.next(key, val)) { System.err.println("Cannot read entry " + i); success = false; break; } else { if (!((key.get() == i) && (val.get() == dist[i]))) { System.err.println("Mismatch! Pos=" + key.get() + ", i=" + i + ", val=" + val.get() + ", dist[i]=" + dist[i]); success = false; } totalseen += val.get(); } } if (success) { if (in.next(key, val)) { System.err.println("Unnecessary lines in recomputed key!"); success = false; } } } finally { in.close(); } int originalTotal = 0; for (int i = 0; i < dist.length; i++) { originalTotal += dist[i]; } System.out.println("Original sum: " + originalTotal); System.out.println("Recomputed sum: " + totalseen); // // Write to "results" whether the test succeeded or not. // Path resultFile = new Path(testdir, "results"); BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(fs.create(resultFile))); try { bw.write("Success=" + success + "\n"); System.out.println("Success=" + success); } finally { bw.close(); } assertTrue("testMapRed failed", success); fs.delete(testdir, true);
5287 -1009652435apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Checking if a Mapper.map within a Runnable has generated a RuntimeException. If so we rethrow it to force an abort of the Map operation thus keeping the semantics of the default implementation. SATD_ADDED configure(JobConf) public void configure(JobConf jobConf) int numberOfThreads = jobConf.getInt(MultithreadedMapper.NUM_THREADS, 10); if (LOG.isDebugEnabled()) { LOG.debug("Configuring jobConf " + jobConf.getJobName() + " to use " + numberOfThreads + " threads"); } this.job = jobConf; // increment processed counter only if skipping feature is enabled this.incrProcCount = SkipBadRecords.getMapperMaxSkipRecords(job) > 0 && SkipBadRecords.getAutoIncrMapperProcCount(job); this.mapper = ReflectionUtils.newInstance(jobConf.getMapperClass(), jobConf); // Creating a threadpool of the configured size to execute the Mapper // map method in parallel. executorService = new ThreadPoolExecutor(numberOfThreads, numberOfThreads, 0L, TimeUnit.MILLISECONDS, new BlockingArrayQueue(numberOfThreads));
5286 -1009652436apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Checking if a Mapper.map within a Runnable has generated an IOException. If so we rethrow it to force an abort of the Map operation thus keeping the semantics of the default implementation. SATD_ADDED configure(JobConf) public void configure(JobConf jobConf) int numberOfThreads = jobConf.getInt(MultithreadedMapper.NUM_THREADS, 10); if (LOG.isDebugEnabled()) { LOG.debug("Configuring jobConf " + jobConf.getJobName() + " to use " + numberOfThreads + " threads"); } this.job = jobConf; // increment processed counter only if skipping feature is enabled this.incrProcCount = SkipBadRecords.getMapperMaxSkipRecords(job) > 0 && SkipBadRecords.getAutoIncrMapperProcCount(job); this.mapper = ReflectionUtils.newInstance(jobConf.getMapperClass(), jobConf); // Creating a threadpool of the configured size to execute the Mapper // map method in parallel. executorService = new ThreadPoolExecutor(numberOfThreads, numberOfThreads, 0L, TimeUnit.MILLISECONDS, new BlockingArrayQueue(numberOfThreads));
5285 -1009652437apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None TODO: figure out why length would be 0 SATD_ADDED getProgress() public float getProgress() throws IOException // TODO: figure out why length would be 0 if (length == 0) return 1; return pos / length;
5284 -1009652438apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None TODO bad dependency SATD_ADDED save(File, FSNamesystem, FSImageCompression) void save(File newFile, FSNamesystem sourceNamesystem, FSImageCompression compression) throws IOException checkNotSaved(); FSDirectory fsDir = sourceNamesystem.dir; long startTime = now(); // // Write out data // MessageDigest digester = MD5Hash.getDigester(); FileOutputStream fout = new FileOutputStream(newFile); DigestOutputStream fos = new DigestOutputStream(fout, digester); DataOutputStream out = new DataOutputStream(fos); try { out.writeInt(FSConstants.LAYOUT_VERSION); // TODO bad dependency out.writeInt(sourceNamesystem.getFSImage().getStorage().getNamespaceID()); out.writeLong(fsDir.rootDir.numItemsInTree()); out.writeLong(sourceNamesystem.getGenerationStamp()); // write compression info and set up compressed stream out = compression.writeHeaderAndWrapStream(fos); LOG.info("Saving image file " + newFile + " using " + compression); byte[] byteStore = new byte[4 * FSConstants.MAX_PATH_LENGTH]; ByteBuffer strbuf = ByteBuffer.wrap(byteStore); // save the root FSImageSerialization.saveINode2Image(fsDir.rootDir, out); // save the rest of the nodes saveImage(strbuf, fsDir.rootDir, out); // save files under construction sourceNamesystem.saveFilesUnderConstruction(out); sourceNamesystem.saveSecretManagerState(out); strbuf = null; out.flush(); fout.getChannel().force(true); } finally { out.close(); } saved = true; // set md5 of the saved image savedDigest = new MD5Hash(digester.digest()); LOG.info("Image file of size " + newFile.length() + " saved in " + (now() - startTime) / 1000 + " seconds.");
5283 -1009652439apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None scan all policies once every 5 second SATD_ADDED mySetup(int, int) protected void mySetup(int stripeLength, int timeBeforeHar) throws Exception // Make sure data directory exists new File(TEST_DIR).mkdirs(); conf = new Configuration(); conf.set("raid.config.file", CONFIG_FILE); conf.setBoolean("raid.config.reload", true); conf.setLong("raid.config.reload.interval", RELOAD_INTERVAL); // scan all policies once every 5 second conf.setLong("raid.policy.rescan.interval", 5000); // make all deletions not go through Trash conf.set("fs.shell.delete.classname", "org.apache.hadoop.hdfs.DFSClient"); // do not use map-reduce cluster for Raiding conf.set("raid.classname", "org.apache.hadoop.raid.LocalRaidNode"); conf.set("raid.server.address", "localhost:0"); conf.setInt("hdfs.raid.stripeLength", stripeLength); conf.set("hdfs.raid.locations", "/destraid"); conf.setBoolean("dfs.permissions", false); dfs = new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DATANODES).build(); dfs.waitActive(); fileSys = dfs.getFileSystem(); namenode = fileSys.getUri().toString(); FileSystem.setDefaultUri(conf, namenode); mr = new MiniMRCluster(4, namenode, 3); jobTrackerName = "localhost:" + mr.getJobTrackerPort(); hftp = "hftp://localhost.localdomain:" + dfs.getNameNodePort(); FileSystem.setDefaultUri(conf, namenode); conf.set("mapred.job.tracker", jobTrackerName); FileWriter fileWriter = new FileWriter(CONFIG_FILE); fileWriter.write("\n"); String str = " " + " " + " " + "xor " + " /destraid " + " " + "targetReplication " + "1 " + "after RAIDing, decrease the replication factor of a file to this value." + " " + " " + " " + "metaReplication " + "1 " + " replication factor of parity file" + " " + " " + " " + "modTimePeriod " + "2000 " + " time (milliseconds) after a file is modified to make it " + "a candidate for RAIDing " + " " + " "; if (timeBeforeHar >= 0) { str += " " + "time_before_har " + "" + timeBeforeHar + " " + " amount of time waited before har'ing parity files" + " " + " "; } str += "" + "" + ""; fileWriter.write(str); fileWriter.close();
5282 -1009652440apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None We don't want to run the failing map task 4 times. So we run it once and check if all the subprocesses are killed properly. SATD_ADDED runKillingJobAndValidate(JobTracker, JobConf) private static void runKillingJobAndValidate(JobTracker jt, JobConf conf) throws IOException conf.setJobName("testkilljobsubprocesses"); conf.setMapperClass(KillingMapperWithChildren.class); RunningJob job = runJobAndSetProcessHandle(jt, conf); // kill the job now job.killJob(); while (job.cleanupProgress() == 0.0f) { try { Thread.sleep(100); } catch (InterruptedException ie) { LOG.warn("sleep is interrupted:" + ie); break; } } validateKillingSubprocesses(job, conf); // Checking the Job status assertEquals(job.getJobState(), JobStatus.KILLED);
5281 -1009652441apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Hack for local FS that does not have the concept of a 'mounting point' SATD_ADDED testCustomFile() public void testCustomFile() throws Exception Path inDir = new Path("testing/fileoutputformat/input"); Path outDir = new Path("testing/fileoutputformat/output"); // Hack for local FS that does not have the concept of a 'mounting point' if (isLocalFS()) { String localPathRoot = System.getProperty("test.build.data", "/tmp").replace(' ', '+'); inDir = new Path(localPathRoot, inDir); outDir = new Path(localPathRoot, outDir); } JobConf conf = createJobConf(); FileSystem fs = FileSystem.get(conf); fs.delete(outDir, true); if (!fs.mkdirs(inDir)) { throw new IOException("Mkdirs failed to create " + inDir.toString()); } DataOutputStream file = fs.create(new Path(inDir, "part-0")); file.writeBytes("a\nb\n\nc\nd\ne"); file.close(); file = fs.create(new Path(inDir, "part-1")); file.writeBytes("a\nb\n\nc\nd\ne"); file.close(); conf.setJobName("fof"); conf.setInputFormat(TextInputFormat.class); conf.setMapOutputKeyClass(LongWritable.class); conf.setMapOutputValueClass(Text.class); conf.setOutputFormat(TextOutputFormat.class); conf.setOutputKeyClass(LongWritable.class); conf.setOutputValueClass(Text.class); conf.setMapperClass(TestMap.class); conf.setReducerClass(TestReduce.class); FileInputFormat.setInputPaths(conf, inDir); FileOutputFormat.setOutputPath(conf, outDir); JobClient jc = new JobClient(conf); RunningJob job = jc.submitJob(conf); while (!job.isComplete()) { Thread.sleep(100); } assertTrue(job.isSuccessful()); boolean map0 = false; boolean map1 = false; boolean reduce = false; FileStatus[] statuses = fs.listStatus(outDir); for (FileStatus status : statuses) { map0 = map0 || status.getPath().getName().equals("test-m-00000"); map1 = map1 || status.getPath().getName().equals("test-m-00001"); reduce = reduce || status.getPath().getName().equals("test-r-00000"); } assertTrue(map0); assertTrue(map1); assertTrue(reduce);
5280 -1009652442apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Now we need to generate the random numbers according to the above distribution. We create a lot of map tasks, each of which takes at least one \"line\" of the distribution. (That is, a certain number X is to be generated Y number of times.) A map task emits Y key/val pairs. The val is X. The key is a randomly-generated number. The reduce task gets its input sorted by key. That is, sorted in random order. It then emits a single line of text that for the given values. It does not emit the key. Because there's just one reduce task, we emit a single big file of random numbers. SATD_ADDED launch() public static void launch() throws Exception // // Generate distribution of ints. This is the answer key. // int countsToGo = counts; int[] dist = new int[range]; for (int i = 0; i < range; i++) { double avgInts = (1.0 * countsToGo) / (range - i); dist[i] = (int) Math.max(0, Math.round(avgInts + (Math.sqrt(avgInts) * r.nextGaussian()))); countsToGo -= dist[i]; } if (countsToGo > 0) { dist[dist.length - 1] += countsToGo; } // // Write the answer key to a file. // FileSystem fs = FileSystem.get(conf); Path testdir = new Path("mapred.loadtest"); if (!fs.mkdirs(testdir)) { throw new IOException("Mkdirs failed to create directory " + testdir.toString()); } Path randomIns = new Path(testdir, "genins"); if (!fs.mkdirs(randomIns)) { throw new IOException("Mkdirs failed to create directory " + randomIns.toString()); } Path answerkey = new Path(randomIns, "answer.key"); SequenceFile.Writer out = SequenceFile.createWriter(fs, conf, answerkey, RecInt.class, RecInt.class, CompressionType.NONE); try { for (int i = 0; i < range; i++) { RecInt k = new RecInt(); RecInt v = new RecInt(); k.setData(i); v.setData(dist[i]); out.append(k, v); } } finally { out.close(); } // // Now we need to generate the random numbers according to // the above distribution. // // We create a lot of map tasks, each of which takes at least // one "line" of the distribution. (That is, a certain number // X is to be generated Y number of times.) // // A map task emits Y key/val pairs. The val is X. The key // is a randomly-generated number. // // The reduce task gets its input sorted by key. That is, sorted // in random order. It then emits a single line of text that // for the given values. It does not emit the key. // // Because there's just one reduce task, we emit a single big // file of random numbers. // Path randomOuts = new Path(testdir, "genouts"); fs.delete(randomOuts, true); JobConf genJob = new JobConf(conf, TestRecordMR.class); FileInputFormat.setInputPaths(genJob, randomIns); genJob.setInputFormat(SequenceFileInputFormat.class); genJob.setMapperClass(RandomGenMapper.class); FileOutputFormat.setOutputPath(genJob, randomOuts); genJob.setOutputKeyClass(RecInt.class); genJob.setOutputValueClass(RecString.class); genJob.setOutputFormat(SequenceFileOutputFormat.class); genJob.setReducerClass(RandomGenReducer.class); genJob.setNumReduceTasks(1); JobClient.runJob(genJob); // // Next, we read the big file in and regenerate the // original map. It's split into a number of parts. // (That number is 'intermediateReduces'.) // // We have many map tasks, each of which read at least one // of the output numbers. For each number read in, the // map task emits a key/value pair where the key is the // number and the value is "1". // // We have a single reduce task, which receives its input // sorted by the key emitted above. For each key, there will // be a certain number of "1" values. The reduce task sums // these values to compute how many times the given key was // emitted. // // The reduce task then emits a key/val pair where the key // is the number in question, and the value is the number of // times the key was emitted. This is the same format as the // original answer key (except that numbers emitted zero times // will not appear in the regenerated key.) The answer set // is split into a number of pieces. A final MapReduce job // will merge them. // // There's not really a need to go to 10 reduces here // instead of 1. But we want to test what happens when // you have multiple reduces at once. // int intermediateReduces = 10; Path intermediateOuts = new Path(testdir, "intermediateouts"); fs.delete(intermediateOuts, true); JobConf checkJob = new JobConf(conf, TestRecordMR.class); FileInputFormat.setInputPaths(checkJob, randomOuts); checkJob.setInputFormat(SequenceFileInputFormat.class); checkJob.setMapperClass(RandomCheckMapper.class); FileOutputFormat.setOutputPath(checkJob, intermediateOuts); checkJob.setOutputKeyClass(RecInt.class); checkJob.setOutputValueClass(RecString.class); checkJob.setOutputFormat(SequenceFileOutputFormat.class); checkJob.setReducerClass(RandomCheckReducer.class); checkJob.setNumReduceTasks(intermediateReduces); JobClient.runJob(checkJob); // // OK, now we take the output from the last job and // merge it down to a single file. The map() and reduce() // functions don't really do anything except reemit tuples. // But by having a single reduce task here, we end up merging // all the files. // Path finalOuts = new Path(testdir, "finalouts"); fs.delete(finalOuts, true); JobConf mergeJob = new JobConf(conf, TestRecordMR.class); FileInputFormat.setInputPaths(mergeJob, intermediateOuts); mergeJob.setInputFormat(SequenceFileInputFormat.class); mergeJob.setMapperClass(MergeMapper.class); FileOutputFormat.setOutputPath(mergeJob, finalOuts); mergeJob.setOutputKeyClass(RecInt.class); mergeJob.setOutputValueClass(RecInt.class); mergeJob.setOutputFormat(SequenceFileOutputFormat.class); mergeJob.setReducerClass(MergeReducer.class); mergeJob.setNumReduceTasks(1); JobClient.runJob(mergeJob); // // Finally, we compare the reconstructed answer key with the // original one. Remember, we need to ignore zero-count items // in the original key. // boolean success = true; Path recomputedkey = new Path(finalOuts, "part-00000"); SequenceFile.Reader in = new SequenceFile.Reader(fs, recomputedkey, conf); int totalseen = 0; try { RecInt key = new RecInt(); RecInt val = new RecInt(); for (int i = 0; i < range; i++) { if (dist[i] == 0) { continue; } if (!in.next(key, val)) { System.err.println("Cannot read entry " + i); success = false; break; } else { if (!((key.getData() == i) && (val.getData() == dist[i]))) { System.err.println("Mismatch! Pos=" + key.getData() + ", i=" + i + ", val=" + val.getData() + ", dist[i]=" + dist[i]); success = false; } totalseen += val.getData(); } } if (success) { if (in.next(key, val)) { System.err.println("Unnecessary lines in recomputed key!"); success = false; } } } finally { in.close(); } int originalTotal = 0; for (int i = 0; i < dist.length; i++) { originalTotal += dist[i]; } System.out.println("Original sum: " + originalTotal); System.out.println("Recomputed sum: " + totalseen); // // Write to "results" whether the test succeeded or not. // Path resultFile = new Path(testdir, "results"); BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(fs.create(resultFile))); try { bw.write("Success=" + success + "\n"); System.out.println("Success=" + success); } finally { bw.close(); } fs.delete(testdir, true);
5279 -1009652443apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Since we're creating a new UserGroupInformation here, we know that no future RPC proxies will be able to re-use the same connection. And usages of this proxy tend to be one-off calls. This is a temporary fix: callers should really achieve this by using RPC.stopProxy() on the resulting object, but this is currently not working in trunk. See the discussion on HDFS-1965. SATD_ADDED createClientDatanodeProtocolProxy(DatanodeID, Configuration, int, LocatedBlock) static ClientDatanodeProtocol createClientDatanodeProtocolProxy(DatanodeID datanodeid, Configuration conf, int socketTimeout, LocatedBlock locatedBlock) throws IOException InetSocketAddress addr = NetUtils.createSocketAddr(datanodeid.getHost() + ":" + datanodeid.getIpcPort()); if (ClientDatanodeProtocol.LOG.isDebugEnabled()) { ClientDatanodeProtocol.LOG.debug("ClientDatanodeProtocol addr=" + addr); } // Since we're creating a new UserGroupInformation here, we know that no // future RPC proxies will be able to re-use the same connection. And // usages of this proxy tend to be one-off calls. // // This is a temporary fix: callers should really achieve this by using // RPC.stopProxy() on the resulting object, but this is currently not // working in trunk. See the discussion on HDFS-1965. Configuration confWithNoIpcIdle = new Configuration(conf); confWithNoIpcIdle.setInt(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECTION_MAXIDLETIME_KEY, 0); UserGroupInformation ticket = UserGroupInformation.createRemoteUser(locatedBlock.getBlock().getLocalBlock().toString()); ticket.addToken(locatedBlock.getBlockToken()); return (ClientDatanodeProtocol) RPC.getProxy(ClientDatanodeProtocol.class, ClientDatanodeProtocol.versionID, addr, ticket, confWithNoIpcIdle, NetUtils.getDefaultSocketFactory(conf), socketTimeout);
5278 -1009652444apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Its alright for a dir not to exist, or to exist (properly accessible) and be completely empty. SATD_ADDED format(Configuration, boolean) private static boolean format(Configuration conf, boolean isConfirmationNeeded) throws IOException if (!conf.getBoolean(DFSConfigKeys.DFS_NAMENODE_SUPPORT_ALLOW_FORMAT_KEY, DFSConfigKeys.DFS_NAMENODE_SUPPORT_ALLOW_FORMAT_DEFAULT)) { throw new IOException("The option " + DFSConfigKeys.DFS_NAMENODE_SUPPORT_ALLOW_FORMAT_KEY + " is set to false for this filesystem, so it " + "cannot be formatted. You will need to set " + DFSConfigKeys.DFS_NAMENODE_SUPPORT_ALLOW_FORMAT_KEY + " parameter " + "to true in order to format this filesystem"); } Collection dirsToFormat = FSNamesystem.getNamespaceDirs(conf); Collection editDirsToFormat = FSNamesystem.getNamespaceEditsDirs(conf); for (Iterator it = dirsToFormat.iterator(); it.hasNext(); ) { File curDir = new File(it.next().getPath()); // Its alright for a dir not to exist, or to exist (properly accessible) // and be completely empty. if (!curDir.exists() || (curDir.isDirectory() && FileUtil.listFiles(curDir).length == 0)) continue; if (isConfirmationNeeded) { if (!confirmPrompt("Re-format filesystem in " + curDir + " ?")) { System.err.println("Format aborted in " + curDir); return true; } } } // if clusterID is not provided - see if you can find the current one String clusterId = StartupOption.FORMAT.getClusterId(); if (clusterId == null || clusterId.equals("")) { // Generate a new cluster id clusterId = NNStorage.newClusterID(); } System.out.println("Formatting using clusterid: " + clusterId); FSImage fsImage = new FSImage(dirsToFormat, editDirsToFormat); FSNamesystem nsys = new FSNamesystem(fsImage, conf); nsys.dir.fsImage.getStorage().format(clusterId); return false;
5277 -1009652445apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None set service-level authorization security policy SATD_ADDED initialize(Configuration) protected void initialize(Configuration conf) throws IOException InetSocketAddress socAddr = getRpcServerAddress(conf); UserGroupInformation.setConfiguration(conf); SecurityUtil.login(conf, DFSConfigKeys.DFS_NAMENODE_KEYTAB_FILE_KEY, DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY, socAddr.getHostName()); int handlerCount = conf.getInt(DFSConfigKeys.DFS_DATANODE_HANDLER_COUNT_KEY, DFSConfigKeys.DFS_DATANODE_HANDLER_COUNT_DEFAULT); NameNode.initMetrics(conf, this.getRole()); loadNamesystem(conf); // create rpc server InetSocketAddress dnSocketAddr = getServiceRpcServerAddress(conf); if (dnSocketAddr != null) { int serviceHandlerCount = conf.getInt(DFSConfigKeys.DFS_NAMENODE_SERVICE_HANDLER_COUNT_KEY, DFSConfigKeys.DFS_NAMENODE_SERVICE_HANDLER_COUNT_DEFAULT); this.serviceRpcServer = RPC.getServer(NamenodeProtocols.class, this, dnSocketAddr.getHostName(), dnSocketAddr.getPort(), serviceHandlerCount, false, conf, namesystem.getDelegationTokenSecretManager()); this.serviceRPCAddress = this.serviceRpcServer.getListenerAddress(); setRpcServiceServerAddress(conf); } this.server = RPC.getServer(NamenodeProtocols.class, this, socAddr.getHostName(), socAddr.getPort(), handlerCount, false, conf, namesystem.getDelegationTokenSecretManager()); // set service-level authorization security policy if (serviceAuthEnabled = conf.getBoolean(CommonConfigurationKeys.HADOOP_SECURITY_AUTHORIZATION, false)) { this.server.refreshServiceAcl(conf, new HDFSPolicyProvider()); if (this.serviceRpcServer != null) { this.serviceRpcServer.refreshServiceAcl(conf, new HDFSPolicyProvider()); } } // The rpc-server port can be ephemeral... ensure we have the correct info this.rpcAddress = this.server.getListenerAddress(); setRpcServerAddress(conf); activate(conf); LOG.info(getRole() + " up at: " + rpcAddress); if (serviceRPCAddress != null) { LOG.info(getRole() + " service server is up at: " + serviceRPCAddress); }
5276 -1009652446apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Private Distributed cache will always be stored under mapre.local.dir/taskTracker//distcache Checking for username directory to check if it has the proper permissions SATD_ADDED testDistributedCache() public void testDistributedCache() throws Exception Configuration conf = new Configuration(cluster.getConf()); JTProtocol wovenClient = cluster.getJTClient().getProxy(); // This counter will check for count of a loop, // which might become infinite. int count = 0; SleepJob job = new SleepJob(); job.setConf(conf); Job slpJob = job.createJob(5, 1, 1000, 1000, 100, 100); DistributedCache.createSymlink(conf); URI uri = URI.create(uriPath); DistributedCache.addCacheFile(uri, conf); JobConf jconf = new JobConf(conf); // Controls the job till all verification is done FinishTaskControlAction.configureControlActionForJob(conf); // Submitting the job slpJob.submit(); RunningJob rJob = cluster.getJTClient().getClient().getJob(org.apache.hadoop.mapred.JobID.downgrade(slpJob.getJobID())); JobStatus[] jobStatus = client.getAllJobs(); String userName = jobStatus[0].getUsername(); TTClient tClient = null; JobInfo jInfo = wovenClient.getJobInfo(rJob.getID()); LOG.info("jInfo is :" + jInfo); // Assert if jobInfo is null Assert.assertNotNull("jobInfo is null", jInfo); // Wait for the job to start running. count = 0; while (jInfo.getStatus().getRunState() != JobStatus.RUNNING) { UtilsForTests.waitFor(10000); count++; jInfo = wovenClient.getJobInfo(rJob.getID()); // If the count goes beyond a point, then Assert; This is to avoid // infinite loop under unforeseen circumstances. if (count > 10) { Assert.fail("job has not reached running state for more than" + "100 seconds. Failing at this point"); } } LOG.info("job id is :" + rJob.getID().toString()); TaskInfo[] taskInfos = cluster.getJTClient().getProxy().getTaskInfo(rJob.getID()); boolean distCacheFileIsFound; for (TaskInfo taskInfo : taskInfos) { distCacheFileIsFound = false; String[] taskTrackers = taskInfo.getTaskTrackers(); for (String taskTracker : taskTrackers) { // Getting the exact FQDN of the tasktracker from // the tasktracker string. taskTracker = UtilsForTests.getFQDNofTT(taskTracker); tClient = cluster.getTTClient(taskTracker); String[] localDirs = tClient.getMapredLocalDirs(); int distributedFileCount = 0; String localDirOnly = null; boolean FileNotPresentForThisDirectoryPath = false; // Go to every single path for (String localDir : localDirs) { FileNotPresentForThisDirectoryPath = false; localDirOnly = localDir; // Public Distributed cache will always be stored under // mapred.local.dir/tasktracker/archive localDirOnly = localDir + Path.SEPARATOR + TaskTracker.SUBDIR + Path.SEPARATOR + userName; // Private Distributed cache will always be stored under // mapre.local.dir/taskTracker//distcache // Checking for username directory to check if it has the // proper permissions localDir = localDir + Path.SEPARATOR + TaskTracker.getPrivateDistributedCacheDir(userName); FileStatus fileStatusMapredLocalDirUserName = null; try { fileStatusMapredLocalDirUserName = tClient.getFileStatus(localDirOnly, true); } catch (Exception e) { LOG.info("LocalDirOnly :" + localDirOnly + " not found"); FileNotPresentForThisDirectoryPath = true; } // File will only be stored under one of the mapred.lcoal.dir // If other paths were hit, just continue if (FileNotPresentForThisDirectoryPath) continue; Path pathMapredLocalDirUserName = fileStatusMapredLocalDirUserName.getPath(); FsPermission fsPermMapredLocalDirUserName = fileStatusMapredLocalDirUserName.getPermission(); Assert.assertTrue("Directory Permission is not 700", fsPermMapredLocalDirUserName.equals(new FsPermission("700"))); // Get file status of all the directories // and files under that path. FileStatus[] fileStatuses = tClient.listStatus(localDir, true, true); for (FileStatus fileStatus : fileStatuses) { Path path = fileStatus.getPath(); LOG.info("path is :" + path.toString()); // Checking if the received path ends with // the distributed filename distCacheFileIsFound = (path.toString()).endsWith(distributedFileName); // If file is found, check for its permission. // Since the file is found break out of loop if (distCacheFileIsFound) { LOG.info("PATH found is :" + path.toString()); distributedFileCount++; String filename = path.getName(); FsPermission fsPerm = fileStatus.getPermission(); Assert.assertTrue("File Permission is not 777", fsPerm.equals(new FsPermission("777"))); } } } LOG.info("Distributed File count is :" + distributedFileCount); if (distributedFileCount > 1) { Assert.fail("The distributed cache file is more than one"); } else if (distributedFileCount < 1) Assert.fail("The distributed cache file is less than one"); if (!distCacheFileIsFound) { Assert.assertEquals("The distributed cache file does not exist", distCacheFileIsFound, false); } } // Allow the job to continue through MR control job. for (TaskInfo taskInfoRemaining : taskInfos) { FinishTaskControlAction action = new FinishTaskControlAction(TaskID.downgrade(taskInfoRemaining.getTaskID())); Collection tts = cluster.getTTClients(); for (TTClient cli : tts) { cli.getProxy().sendAction(action); } } // Killing the job because all the verification needed // for this testcase is completed. rJob.killJob(); }
5275 -1009652447apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Move decommissioned datanodes to the bottom SATD_ADDED initialValue() protected StringBuilder initialValue() return new StringBuilder();
5274 -1009652448apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Validate if DistributedCacheEmulator can handle a JobStory with out Distributed Cache files properly. SATD_ADDED testDistCacheFilesConfiguration() public void testDistCacheFilesConfiguration() throws IOException Configuration conf = new Configuration(); JobConf jobConf = GridmixTestUtils.mrCluster.createJobConf(new JobConf(conf)); Path ioPath = new Path("testDistCacheEmulationConfigurability").makeQualified(GridmixTestUtils.dfs); FileSystem fs = FileSystem.get(jobConf); FileSystem.mkdirs(fs, ioPath, new FsPermission((short) 0777)); // default config dce = createDistributedCacheEmulator(jobConf, ioPath, false); assertTrue("Default configuration of " + DistributedCacheEmulator.GRIDMIX_EMULATE_DISTRIBUTEDCACHE + " is wrong.", dce.shouldEmulateDistCacheLoad()); // Validate if DistributedCacheEmulator can handle a JobStory with out // Distributed Cache files properly. validateJobConfWithOutDCFiles(conf, jobConf); // Validate if Gridmix can configure dist cache files properly if there are // HDFS-based dist cache files and localFS-based dist cache files in trace // for a job. Set old config properties and validate. validateJobConfWithDCFiles(conf, jobConf); // Use new JobConf as JobStory conf and check if configureDistCacheFiles() // doesn't throw NPE when there are dist cache files set but visibilities // are not set. validateWithOutVisibilities();
5273 -1009652449apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Verify if correct exit code is seen when -generate option is missing and distributed cache files are missing in the expected path. SATD_ADDED testSetupGenerateDistCacheData() public void testSetupGenerateDistCacheData() throws IOException, InterruptedException long[] sortedFileSizes = new long[5]; JobConf jobConf = runSetupGenerateDistCacheData(true, sortedFileSizes); validateSetupGenDC(jobConf, sortedFileSizes); // Verify if correct exit code is seen when -generate option is missing and // distributed cache files are missing in the expected path. runSetupGenerateDistCacheData(false, sortedFileSizes);
5272 -1009652450apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Validate the existence of Distributed Cache files list file directly under distributed cache directory SATD_ADDED validateDistCacheData(JobConf, long[]) private void validateDistCacheData(JobConf jobConf, long[] sortedFileSizes) throws FileNotFoundException, IOException Path distCachePath = dce.getDistributedCacheDir(); String filesListFile = jobConf.get(GenerateDistCacheData.GRIDMIX_DISTCACHE_FILE_LIST); FileSystem fs = FileSystem.get(jobConf); // Validate the existence of Distributed Cache files list file directly // under distributed cache directory Path listFile = new Path(filesListFile); assertTrue("Path of Distributed Cache files list file is wrong.", distCachePath.equals(listFile.getParent().makeQualified(fs))); // Delete the dist cache files list file assertTrue("Failed to delete distributed Cache files list file " + listFile, fs.delete(listFile)); List fileSizes = new ArrayList(); for (long size : sortedFileSizes) { fileSizes.add(size); } // validate dist cache files after deleting the 'files list file' validateDistCacheFiles(fileSizes, distCachePath);
5271 -1009652451apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None our target rate is in terms of the maximum number of sample splits, but we accept the possibility of sampling additional splits to hit the target sample keyset SATD_ADDED printUsage() static int printUsage() System.out.println("sampler -r \n" + " [-inFormat ]\n" + " [-keyClass ]\n" + " [-splitRandom | " + " // Sample from random splits at random (general)\n" + " -splitSample | " + " // Sample from first records in splits (random data)\n" + " -splitInterval ]" + " // Sample from splits at intervals (sorted data)"); System.out.println("Default sampler: -splitRandom 0.1 10000 10"); ToolRunner.printGenericCommandUsage(System.out); return -1;
5270 -1009652452apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None initialize buffer to the best guess size: 'chunksPerPacket' calculation here should match the same calculation in DFSClient to make the guess accurate. SATD_ADDED getDataNode() return datanode; DataNode getDataNode()
5268 -1009652454apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None We should never receive a duplicate success/failure/killed status update for the same taskid! This is a safety check, and is addressed better at the TaskTracker to ensure this. SATD_ADDED setMaxTaskAttempts() private void setMaxTaskAttempts() if (isMapTask()) { this.maxTaskAttempts = conf.getMaxMapAttempts(); } else { this.maxTaskAttempts = conf.getMaxReduceAttempts(); }
5267 -1009652455apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None desired range starts after beginning of this har block fix offset to beginning of relevant range (relative to desired file) SATD_ADDED initialize(URI, Configuration) public void initialize(URI name, Configuration conf) throws IOException // decode the name URI underLyingURI = decodeHarURI(name, conf); // we got the right har Path- now check if this is // truly a har filesystem Path harPath = archivePath(new Path(name.getScheme(), name.getAuthority(), name.getPath())); if (harPath == null) { throw new IOException("Invalid path for the Har Filesystem. " + name.toString()); } if (fs == null) { fs = FileSystem.get(underLyingURI, conf); } uri = harPath.toUri(); archivePath = new Path(uri.getPath()); harAuth = getHarAuth(underLyingURI); // check for the underlying fs containing // the index file Path masterIndexPath = new Path(archivePath, "_masterindex"); Path archiveIndexPath = new Path(archivePath, "_index"); if (!fs.exists(masterIndexPath) || !fs.exists(archiveIndexPath)) { throw new IOException("Invalid path for the Har Filesystem. " + "No index file in " + harPath); } metadata = harMetaCache.get(uri); if (metadata != null) { FileStatus mStat = fs.getFileStatus(masterIndexPath); FileStatus aStat = fs.getFileStatus(archiveIndexPath); if (mStat.getModificationTime() != metadata.getMasterIndexTimestamp() || aStat.getModificationTime() != metadata.getArchiveIndexTimestamp()) { // the archive has been overwritten since we last read it // remove the entry from the meta data cache metadata = null; harMetaCache.remove(uri); } } if (metadata == null) { metadata = new HarMetaData(fs, masterIndexPath, archiveIndexPath); metadata.parseMetaData(); harMetaCache.put(uri, metadata); }
5266 -1009652456apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None TODO fix this: make it more sophisticated!!! SATD_ADDED injectCriteria(String) public static boolean injectCriteria(String klassName) boolean trigger = false; // TODO fix this: make it more sophisticated!!! if (generator.nextFloat() < getProbability(klassName)) { trigger = true; } return trigger;
5265 -1009652457apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Set new default probability if specified through a system.property If neither is specified set default probability to DEFAULT_PROB SATD_ADDED injectCriteria(String) public static boolean injectCriteria(String klassName) boolean trigger = false; // TODO fix this: make it more sophisticated!!! if (generator.nextFloat() < getProbability(klassName)) { trigger = true; } return trigger;
5263 -1009652459apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Shouldn't happen since we do implement Clonable SATD_ADDED clone() public Object clone() try { return super.clone(); } catch (CloneNotSupportedException cnse) { // Shouldn't happen since we do implement Clonable throw new InternalError(cnse.toString()); }
5262 -1009652460apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None move over ')' SATD_ADDED getBlock(String, char, char, IntWritable) private static String getBlock(String str, char open, char close, IntWritable index) throws ParseException StringBuilder split = new StringBuilder(); int next = StringUtils.findNext(str, open, StringUtils.ESCAPE_CHAR, index.get(), split); // clear the buffer split.setLength(0); if (next >= 0) { // move over '(' ++next; next = StringUtils.findNext(str, close, StringUtils.ESCAPE_CHAR, next, split); if (next >= 0) { // move over ')' ++next; index.set(next); // found a block return split.toString(); } else { throw new ParseException("Unexpected end of block", next); } } // found nothing return null;
5261 -1009652461apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None move over '(' SATD_ADDED getBlock(String, char, char, IntWritable) private static String getBlock(String str, char open, char close, IntWritable index) throws ParseException StringBuilder split = new StringBuilder(); int next = StringUtils.findNext(str, open, StringUtils.ESCAPE_CHAR, index.get(), split); // clear the buffer split.setLength(0); if (next >= 0) { // move over '(' ++next; next = StringUtils.findNext(str, close, StringUtils.ESCAPE_CHAR, next, split); if (next >= 0) { // move over ')' ++next; index.set(next); // found a block return split.toString(); } else { throw new ParseException("Unexpected end of block", next); } } // found nothing return null;
5260 -1009652129apache/hadoopEli Collins3fd40ae8d0b45d7bf6186fe14851ca87eb9ee3ef None test whether the CPU usage emulator achieves the desired target using desired calls to the underling core engine. SATD_ADDED testEmulationAccuracy(Configuration, FakeCpuUsageEmulatorCore, ResourceCalculatorPlugin, ResourceUsageMetrics, CumulativeCpuUsageEmulatorPlugin, long, long) private static void testEmulationAccuracy(Configuration conf, FakeCpuUsageEmulatorCore fakeCore, ResourceCalculatorPlugin monitor, ResourceUsageMetrics metrics, CumulativeCpuUsageEmulatorPlugin cpuPlugin, long expectedTotalCpuUsage, long expectedTotalNumCalls) throws Exception FakeProgressive fakeProgress = new FakeProgressive(); fakeCore.reset(); cpuPlugin.initialize(conf, metrics, monitor, fakeProgress); int numLoops = 0; while (fakeProgress.getProgress() < 1) { ++numLoops; float progress = (float) numLoops / 100; fakeProgress.setProgress(progress); cpuPlugin.emulate(); } // test if the resource plugin shows the expected invocations assertEquals("Cumulative cpu usage emulator plugin failed (num calls)!", expectedTotalNumCalls, fakeCore.getNumCalls(), 0L); // test if the resource plugin shows the expected usage assertEquals("Cumulative cpu usage emulator plugin failed (total usage)!", expectedTotalCpuUsage, fakeCore.getCpuUsage(), 0L);
5258 -1009652463apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None If there is a problem with the har path, this will let us continue. SATD_ADDED aggregate(Statistics) public void aggregate(Statistics other) this.numRaided += other.numRaided; this.numTooNew += other.numTooNew; this.sizeTooNew += other.sizeTooNew; this.numTooSmall += other.numTooSmall; this.sizeTooSmall += other.sizeTooSmall;
5257 -1009652130apache/hadoopEli Collins3fd40ae8d0b45d7bf6186fe14851ca87eb9ee3ef None test with invalid or missing resource usage value SATD_ADDED testCumulativeCpuUsageEmulatorPlugin() public void testCumulativeCpuUsageEmulatorPlugin() throws Exception Configuration conf = new Configuration(); long targetCpuUsage = 1000L; int unitCpuUsage = 50; // fake progress indicator FakeProgressive fakeProgress = new FakeProgressive(); // fake cpu usage generator FakeCpuUsageEmulatorCore fakeCore = new FakeCpuUsageEmulatorCore(); fakeCore.setUnitUsage(unitCpuUsage); // a cumulative cpu usage emulator with fake core CumulativeCpuUsageEmulatorPlugin cpuPlugin = new CumulativeCpuUsageEmulatorPlugin(fakeCore); // test with invalid or missing resource usage value ResourceUsageMetrics invalidUsage = createMetrics(0); cpuPlugin.initialize(conf, invalidUsage, null, null); // test if disabled cpu emulation plugin's emulate() call is a no-operation // this will test if the emulation plugin is disabled or not int numCallsPre = fakeCore.getNumCalls(); long cpuUsagePre = fakeCore.getCpuUsage(); cpuPlugin.emulate(); int numCallsPost = fakeCore.getNumCalls(); long cpuUsagePost = fakeCore.getCpuUsage(); // test if no calls are made cpu usage emulator core assertEquals("Disabled cumulative CPU usage emulation plugin works!", numCallsPre, numCallsPost); // test if no calls are made cpu usage emulator core assertEquals("Disabled cumulative CPU usage emulation plugin works!", cpuUsagePre, cpuUsagePost); // test with valid resource usage value ResourceUsageMetrics metrics = createMetrics(targetCpuUsage); // fake monitor ResourceCalculatorPlugin monitor = new FakeResourceUsageMonitor(fakeCore); // test with default emulation interval testEmulationAccuracy(conf, fakeCore, monitor, metrics, cpuPlugin, targetCpuUsage, targetCpuUsage / unitCpuUsage); // test with custom value for emulation interval of 20% conf.setFloat(CumulativeCpuUsageEmulatorPlugin.CPU_EMULATION_FREQUENCY, 0.2F); testEmulationAccuracy(conf, fakeCore, monitor, metrics, cpuPlugin, targetCpuUsage, targetCpuUsage / unitCpuUsage); // test if emulation interval boundary is respected (unit usage = 1) // test the case where the current progress is less than threshold // initialize fakeProgress = new FakeProgressive(); fakeCore.reset(); fakeCore.setUnitUsage(1); conf.setFloat(CumulativeCpuUsageEmulatorPlugin.CPU_EMULATION_FREQUENCY, 0.25F); cpuPlugin.initialize(conf, metrics, monitor, fakeProgress); // take a snapshot after the initialization long initCpuUsage = monitor.getCumulativeCpuTime(); long initNumCalls = fakeCore.getNumCalls(); // test with 0 progress testEmulationBoundary(0F, fakeCore, fakeProgress, cpuPlugin, initCpuUsage, initNumCalls, "[no-op, 0 progress]"); // test with 24% progress testEmulationBoundary(0.24F, fakeCore, fakeProgress, cpuPlugin, initCpuUsage, initNumCalls, "[no-op, 24% progress]"); // test with 25% progress // target = 1000ms, target emulation at 25% = 250ms, // weighed target = 1000 * 0.25^4 (we are using progress^4 as the weight) // ~ 4 // but current usage = init-usage = 100, hence expected = 100 testEmulationBoundary(0.25F, fakeCore, fakeProgress, cpuPlugin, initCpuUsage, initNumCalls, "[op, 25% progress]"); // test with 80% progress // target = 1000ms, target emulation at 80% = 800ms, // weighed target = 1000 * 0.25^4 (we are using progress^4 as the weight) // ~ 410 // current-usage = init-usage = 100, hence expected-usage = 410 testEmulationBoundary(0.80F, fakeCore, fakeProgress, cpuPlugin, 410, 410, "[op, 80% progress]"); // now test if the final call with 100% progress ramps up the CPU usage testEmulationBoundary(1F, fakeCore, fakeProgress, cpuPlugin, targetCpuUsage, targetCpuUsage, "[op, 100% progress]"); // test if emulation interval boundary is respected (unit usage = 50) // test the case where the current progress is less than threshold // initialize fakeProgress = new FakeProgressive(); fakeCore.reset(); fakeCore.setUnitUsage(unitCpuUsage); conf.setFloat(CumulativeCpuUsageEmulatorPlugin.CPU_EMULATION_FREQUENCY, 0.40F); cpuPlugin.initialize(conf, metrics, monitor, fakeProgress); // take a snapshot after the initialization initCpuUsage = monitor.getCumulativeCpuTime(); initNumCalls = fakeCore.getNumCalls(); // test with 0 progress testEmulationBoundary(0F, fakeCore, fakeProgress, cpuPlugin, initCpuUsage, initNumCalls, "[no-op, 0 progress]"); // test with 39% progress testEmulationBoundary(0.39F, fakeCore, fakeProgress, cpuPlugin, initCpuUsage, initNumCalls, "[no-op, 39% progress]"); // test with 40% progress // target = 1000ms, target emulation at 40% = 4000ms, // weighed target = 1000 * 0.40^4 (we are using progress^4 as the weight) // ~ 26 // current-usage = init-usage = 100, hence expected-usage = 100 testEmulationBoundary(0.40F, fakeCore, fakeProgress, cpuPlugin, initCpuUsage, initNumCalls, "[op, 40% progress]"); // test with 90% progress // target = 1000ms, target emulation at 90% = 900ms, // weighed target = 1000 * 0.90^4 (we are using progress^4 as the weight) // ~ 657 // current-usage = init-usage = 100, hence expected-usage = 657 but // the fake-core increases in steps of 50, hence final target = 700 testEmulationBoundary(0.90F, fakeCore, fakeProgress, cpuPlugin, 700, 700 / unitCpuUsage, "[op, 90% progress]"); // now test if the final call with 100% progress ramps up the CPU usage testEmulationBoundary(1F, fakeCore, fakeProgress, cpuPlugin, targetCpuUsage, targetCpuUsage / unitCpuUsage, "[op, 100% progress]");
5256 -1009652464apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None Remove all policies where srcPath <= currentSrcPath or matchingPrefixLength is < length(currentSrcPath) The policies remaining are the only ones that could better select a file chosen by the current policy. SATD_ADDED aggregate(Statistics) public void aggregate(Statistics other) this.numRaided += other.numRaided; this.numTooNew += other.numTooNew; this.sizeTooNew += other.sizeTooNew; this.numTooSmall += other.numTooSmall; this.sizeTooSmall += other.sizeTooSmall;
5254 -1009652465apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None We need this semaphore to block when the number of running workitems is equal to the number of threads. FixedThreadPool limits the number of threads, but not the queue size. This way we will limit the memory usage. SATD_ADDED check(FileStatus) boolean check(FileStatus f) throws IOException mp
5253 -1009652132apache/hadoopEli Collins3fd40ae8d0b45d7bf6186fe14851ca87eb9ee3ef None TODO Multi-threading for speedup? SATD_ADDED emulate() public void emulate() throws IOException, InterruptedException if (enabled) { float currentProgress = progress.getProgress(); if (lastSeenProgress < currentProgress && ((currentProgress - lastSeenProgress) >= emulationInterval || currentProgress == 1)) { // Estimate the final cpu usage // // Consider the following // Cl/Cc/Cp : Last/Current/Projected Cpu usage // Pl/Pc/Pp : Last/Current/Projected progress // Then // (Cp-Cc)/(Pp-Pc) = (Cc-Cl)/(Pc-Pl) // Solving this for Cp, we get // Cp = Cc + (1-Pc)*(Cc-Cl)/Pc-Pl) // Note that (Cc-Cl)/(Pc-Pl) is termed as 'rate' in the following // section long currentCpuUsage = monitor.getProcResourceValues().getCumulativeCpuTime(); // estimate the cpu usage rate float rate = (currentCpuUsage - lastSeenCpuUsageCpuUsage) / (currentProgress - lastSeenProgress); long projectedUsage = currentCpuUsage + (long) ((1 - currentProgress) * rate); if (projectedUsage < targetCpuUsage) { // determine the correction factor between the current usage and the // expected usage and add some weight to the target long currentWeighedTarget = (long) (targetCpuUsage * getWeightForProgressInterval(currentProgress)); while (monitor.getProcResourceValues().getCumulativeCpuTime() < currentWeighedTarget) { emulatorCore.compute(); // sleep for 100ms try { Thread.sleep(100); } catch (InterruptedException ie) { String message = "CumulativeCpuUsageEmulatorPlugin got interrupted. Exiting."; throw new RuntimeException(message); } } } // set the last seen progress lastSeenProgress = progress.getProgress(); // set the last seen usage lastSeenCpuUsageCpuUsage = monitor.getProcResourceValues().getCumulativeCpuTime(); } }
5252 -1009652466apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None perhaps we should bail out here... SATD_ADDED fsck() public void fsck() final long startTime = System.currentTimeMillis(); try { String msg = "FSCK started by " + UserGroupInformation.getCurrentUser() + " from " + remoteAddress + " for path " + path + " at " + new Date(); LOG.info(msg); out.println(msg); namenode.getNamesystem().logFsckEvent(path, remoteAddress); final HdfsFileStatus file = namenode.getFileInfo(path); if (file != null) { if (showCorruptFileBlocks) { listCorruptFileBlocks(); return; } Result res = new Result(conf); check(path, file, res); out.println(res); out.println(" Number of data-nodes:\t\t" + totalDatanodes); out.println(" Number of racks:\t\t" + networktopology.getNumOfRacks()); out.println("FSCK ended at " + new Date() + " in " + (System.currentTimeMillis() - startTime + " milliseconds")); // DFSck client scans for the string HEALTHY/CORRUPT to check the status // of file system and return appropriate code. Changing the output // string might break testcases. Also note this must be the last line // of the report. if (res.isHealthy()) { out.print("\n\nThe filesystem under path '" + path + "' " + HEALTHY_STATUS); } else { out.print("\n\nThe filesystem under path '" + path + "' " + CORRUPT_STATUS); } } else { out.print("\n\nPath '" + path + "' " + NONEXISTENT_STATUS); } } catch (Exception e) { String errMsg = "Fsck on path '" + path + "' " + FAILURE_STATUS; LOG.warn(errMsg, e); out.println("FSCK ended at " + new Date() + " in " + (System.currentTimeMillis() - startTime + " milliseconds")); out.println(e.getMessage()); out.print("\n\n" + errMsg); } finally { out.close(); }
5251 -1009652133apache/hadoopEli Collins3fd40ae8d0b45d7bf6186fe14851ca87eb9ee3ef None we want some kind of exponential growth function that gives less weight on lower progress boundaries but high (exact emulation) near progress value of 1. so here is how the current growth function looks like progress weight 0.1 0.0001 0.2 0.0016 0.3 0.0081 0.4 0.0256 0.5 0.0625 0.6 0.1296 0.7 0.2401 0.8 0.4096 0.9 0.6561 1.0 1.000 SATD_ADDED getWeightForProgressInterval(float) private float getWeightForProgressInterval(float progress) // we want some kind of exponential growth function that gives less weight // on lower progress boundaries but high (exact emulation) near progress // value of 1. // so here is how the current growth function looks like // progress weight // 0.1 0.0001 // 0.2 0.0016 // 0.3 0.0081 // 0.4 0.0256 // 0.5 0.0625 // 0.6 0.1296 // 0.7 0.2401 // 0.8 0.4096 // 0.9 0.6561 // 1.0 1.000 return progress * progress * progress * progress;
5249 -1009652134apache/hadoopEli Collins3fd40ae8d0b45d7bf6186fe14851ca87eb9ee3ef None compute the 1% of the total CPU usage desired TODO Make this configurable SATD_ADDED calibrate(ResourceCalculatorPlugin, long) public void calibrate(ResourceCalculatorPlugin monitor, long totalCpuUsage) long initTime = monitor.getProcResourceValues().getCumulativeCpuTime(); long defaultLoopSize = 0; long finalTime = initTime; // TODO Make this configurable while (finalTime - initTime < 100) { // 100 ms ++defaultLoopSize; // perform unit computation performUnitComputation(); finalTime = monitor.getProcResourceValues().getCumulativeCpuTime(); } long referenceRuntime = finalTime - initTime; // time for one loop = (final-time - init-time) / total-loops float timePerLoop = ((float) referenceRuntime) / defaultLoopSize; // compute the 1% of the total CPU usage desired // TODO Make this configurable long onePercent = totalCpuUsage / 100; // num-iterations for 1% = (total-desired-usage / 100) / time-for-one-loop numIterations = Math.max(1, (int) ((float) onePercent / timePerLoop)); System.out.println("Calibration done. Basic computation runtime : " + timePerLoop + " milliseconds. Optimal number of iterations (1%): " + numIterations);
5247 -1009652135apache/hadoopEli Collins3fd40ae8d0b45d7bf6186fe14851ca87eb9ee3ef None TODO Make this configurable 100 ms SATD_ADDED calibrate(ResourceCalculatorPlugin, long) public void calibrate(ResourceCalculatorPlugin monitor, long totalCpuUsage) long initTime = monitor.getProcResourceValues().getCumulativeCpuTime(); long defaultLoopSize = 0; long finalTime = initTime; // TODO Make this configurable while (finalTime - initTime < 100) { // 100 ms ++defaultLoopSize; // perform unit computation performUnitComputation(); finalTime = monitor.getProcResourceValues().getCumulativeCpuTime(); } long referenceRuntime = finalTime - initTime; // time for one loop = (final-time - init-time) / total-loops float timePerLoop = ((float) referenceRuntime) / defaultLoopSize; // compute the 1% of the total CPU usage desired // TODO Make this configurable long onePercent = totalCpuUsage / 100; // num-iterations for 1% = (total-desired-usage / 100) / time-for-one-loop numIterations = Math.max(1, (int) ((float) onePercent / timePerLoop)); System.out.println("Calibration done. Basic computation runtime : " + timePerLoop + " milliseconds. Optimal number of iterations (1%): " + numIterations);
5246 -1009652469apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc None maybe didn't work SATD_ADDED run(String[]) public int run(String[] args) ParsedOutput parsedOpts = null; try { ArgumentParser argHolder = new ArgumentParser(args); parsedOpts = argHolder.parse(); if (parsedOpts.shouldOutputHelp()) { parsedOpts.outputHelp(); return 1; } } catch (Exception e) { LOG.error("Unable to parse arguments due to error: ", e); return 1; } LOG.info("Running with option list " + Helper.stringifyArray(args, " ")); ConfigExtractor config = null; try { ConfigMerger cfgMerger = new ConfigMerger(); Configuration cfg = cfgMerger.getMerged(parsedOpts, new Configuration(base)); if (cfg != null) { config = new ConfigExtractor(cfg); } } catch (Exception e) { LOG.error("Unable to merge config due to error: ", e); return 1; } if (config == null) { LOG.error("Unable to merge config & options!"); return 1; } try { LOG.info("Options are:"); ConfigExtractor.dumpOptions(config); } catch (Exception e) { LOG.error("Unable to dump options due to error: ", e); return 1; } boolean jobOk = false; try { LOG.info("Running job:"); runJob(config); jobOk = true; } catch (Exception e) { LOG.error("Unable to run job due to error: ", e); } if (jobOk) { try { LOG.info("Reporting on job:"); writeReport(config); } catch (Exception e) { LOG.error("Unable to report on job due to error: ", e); } } // attempt cleanup (not critical) boolean cleanUp = getBool(parsedOpts.getValue(ConfigOption.CLEANUP.getOpt())); if (cleanUp) { try { LOG.info("Cleaning up job:"); cleanup(config); } catch (Exception e) { LOG.error("Unable to cleanup job due to error: ", e); } } // all mostly worked if (jobOk) { return 0; } // maybe didn't work return 1;
5245 -1009652136apache/hadoopEli Collins3fd40ae8d0b45d7bf6186fe14851ca87eb9ee3ef None TODO can this be configurable too. Users/emulators should be able to pick and choose what MATH operations to run. Example : BASIC : ADD, SUB, MUL, DIV ADV : SQRT, SIN, COSIN.. COMPO : (BASIC/ADV)* Also define input generator. For now we can use the random number generator. Later this can be changed to accept multiple sources. SATD_ADDED performUnitComputation() protected void performUnitComputation() // TODO can this be configurable too. Users/emulators should be able to // pick and choose what MATH operations to run. // Example : // BASIC : ADD, SUB, MUL, DIV // ADV : SQRT, SIN, COSIN.. // COMPO : (BASIC/ADV)* // Also define input generator. For now we can use the random number // generator. Later this can be changed to accept multiple sources. int randomData = random.nextInt(); int randomDataCube = randomData * randomData * randomData; double randomDataCubeRoot = Math.cbrt(randomData); returnValue = Math.log(Math.tan(randomDataCubeRoot * Math.exp(randomDataCube)) * Math.sqrt(randomData));
5242 -1009652471apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc move to current trash move to current trash FILE_PATH_CHANGED moveToAppropriateTrash(FileSystem, Path, Configuration) public static boolean moveToAppropriateTrash(FileSystem fs, Path p, Configuration conf) throws IOException Path fullyResolvedPath = fs.resolvePath(p); Trash trash = new Trash(FileSystem.get(fullyResolvedPath.toUri(), conf), conf); boolean success = trash.moveToTrash(fullyResolvedPath); if (success) { System.out.println("Moved: '" + p + "' to trash at: " + trash.getCurrentTrashDir()); } return success;
5241 -1009652484apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc TODO: commands should implement a -f to enable this TODO: commands should implement a -f to enable this FILE_PATH_CHANGED setOverwrite(boolean) protected void setOverwrite(boolean flag) overwrite = flag;
5240 -1009652510apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc this is an unexpected condition, so dump the whole exception since it's probably a nasty internal error where the backtrace would be useful this is an unexpected condition, so dump the whole exception since it's probably a nasty internal error where the backtrace would be useful FILE_PATH_CHANGED displayError(Exception) public void displayError(Exception e) // build up a list of exceptions that occurred exceptions.add(e); String errorMessage = e.getLocalizedMessage(); if (errorMessage == null) { // this is an unexpected condition, so dump the whole exception since // it's probably a nasty internal error where the backtrace would be // useful errorMessage = StringUtils.stringifyException(e); LOG.debug(errorMessage); } else { errorMessage = errorMessage.split("\n", 2)[0]; } displayError(errorMessage);
5239 -1009652512apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc TODO: this really should be iterative TODO: this really should be iterative FILE_PATH_CHANGED processPaths(PathData, PathData...) protected void processPaths(PathData parent, PathData... items) throws IOException // TODO: this really should be iterative for (PathData item : items) { try { processPath(item); if (recursive && item.stat.isDirectory()) { recursePath(item); } } catch (IOException e) { displayError(e); } }
5238 -1009652551apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc The following if clause handles the following case: Assume the following scenario in BZip2 compressed stream where . represent compressed data. .....[48 bit Block].....[48 bit Block].....[48 bit Block]... ........................[47 bits][1 bit].....[48 bit Block]... ................................^[Assume a Byte alignment here] ........................................^^[current position of stream] .....................^^[We go back 10 Bytes in stream and find a Block marker] ........................................^^[We align at wrong position!] ...........................................................^^[While this pos is correct] The following if clause handles the following case: Assume the following scenario in BZip2 compressed stream where . represent compressed data. .....[48 bit Block].....[48 bit Block].....[48 bit Block]... ........................[47 bits][1 bit].....[48 bit Block]... ................................^[Assume a Byte alignment here] ........................................^^[current position of stream] .....................^^[We go back 10 Bytes in stream and find a Block marker] ........................................^^[We align at wrong position!] ...........................................................^^[While this pos is correct] FILE_PATH_CHANGED createOutputStream(OutputStream) public CompressionOutputStream createOutputStream(OutputStream out) throws IOException return new BZip2CompressionOutputStream(out);
5237 -1009652538apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc Flip to the newly parsed permissions Flip to the newly parsed permissions FILE_PATH_CHANGED refresh(Configuration, PolicyProvider) public synchronized void refresh(Configuration conf, PolicyProvider provider) // Get the system property 'hadoop.policy.file' String policyFile = System.getProperty("hadoop.policy.file", HADOOP_POLICY_FILE); // Make a copy of the original config, and load the policy file Configuration policyConf = new Configuration(conf); policyConf.addResource(policyFile); final Map, AccessControlList> newAcls = new IdentityHashMap, AccessControlList>(); // Parse the config file Service[] services = provider.getServices(); if (services != null) { for (Service service : services) { AccessControlList acl = new AccessControlList(policyConf.get(service.getServiceKey(), AccessControlList.WILDCARD_ACL_VALUE)); newAcls.put(service.getProtocol(), acl); } } // Flip to the newly parsed permissions protocolToAcl = newAcls;
5236 -1009652520apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc For efficient implementation, there's no way around the following massive code duplication. For efficient implementation, there's no way around the following massive code duplication. FILE_PATH_CHANGED writeBooleanArray(DataOutput) private void writeBooleanArray(DataOutput out) throws IOException boolean[] v = (boolean[]) value; for (int i = 0; i < length; i++) out.writeBoolean(v[i]);
5234 -1009652500apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc The implementors of RawLocalFileSystem were trying to be very smart. They implement FileStatus#getOwener lazily -- the object returned is really a RawLocalFileSystem that expect the FileStatus#getPath to be unchanged so that it can get owner when needed. Hence we need to interpose a new ViewFsFileStatus that works around. The implementors of RawLocalFileSystem were trying to be very smart. They implement FileStatus#getOwener lazily -- the object returned is really a RawLocalFileSystem that expect the FileStatus#getPath to be unchanged so that it can get owner when needed. Hence we need to interpose a new ViewFsFileStatus that works around. FILE_PATH_CHANGED readOnlyMountTable(String, String) static AccessControlException readOnlyMountTable(final String operation, final String p) return new AccessControlException("InternalDir of ViewFileSystem is readonly; operation=" + operation + "Path=" + p);
5233 -1009652497apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc this is the stupid semantics of FileSystem this is the stupid semantics of FileSystem FILE_PATH_CHANGED mkdirs(Path, FsPermission) public boolean mkdirs(Path dir, FsPermission permission) throws AccessControlException, FileAlreadyExistsException if (theInternalDir.isRoot & dir == null) { throw new FileAlreadyExistsException("/ already exits"); } // Note dir starts with / if (theInternalDir.children.containsKey(dir.toString().substring(1))) { // this is the stupid semantics of FileSystem return true; } throw readOnlyMountTable("mkdirs", dir);
5232 -1009652498apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc The implementors of RawLocalFileSystem were trying to be very smart. They implement FileStatus#getOwener lazily -- the object returned is really a RawLocalFileSystem that expect the FileStatus#getPath to be unchanged so that it can get owner when needed. Hence we need to interpose a new ViewFileSystemFileStatus that works around. The implementors of RawLocalFileSystem were trying to be very smart. They implement FileStatus#getOwener lazily -- the object returned is really a RawLocalFileSystem that expect the FileStatus#getPath to be unchanged so that it can get owner when needed. Hence we need to interpose a new ViewFileSystemFileStatus that works around. FILE_PATH_CHANGED readOnlyMountTable(String, String) static AccessControlException readOnlyMountTable(final String operation, final String p) return new AccessControlException("InternalDir of ViewFileSystem is readonly; operation=" + operation + "Path=" + p);
5231 -1009652592apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc Workaround end Workaround end FILE_PATH_CHANGED createBaseListener(Configuration) public Connector createBaseListener(Configuration conf) throws IOException return HttpServer.createDefaultChannelConnector();
5230 -1009652593apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc Workaround to handle the problem reported in HADOOP-4744 Workaround to handle the problem reported in HADOOP-4744 FILE_PATH_CHANGED createBaseListener(Configuration) public Connector createBaseListener(Configuration conf) throws IOException return HttpServer.createDefaultChannelConnector();
5229 -1009652559apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc Verify fix of bug identified in HADOOP-6004 Verify fix of bug identified in HADOOP-6004 FILE_PATH_CHANGED testDeserialization() public void testDeserialization() throws IOException // Create a test BlockLocation String[] names = { "one", "two" }; String[] hosts = { "three", "four" }; String[] topologyPaths = { "five", "six" }; long offset = 25l; long length = 55l; BlockLocation bl = new BlockLocation(names, hosts, topologyPaths, offset, length); DataOutputBuffer dob = new DataOutputBuffer(); // Serialize it try { bl.write(dob); } catch (IOException e) { fail("Unable to serialize data: " + e.getMessage()); } byte[] bytes = dob.getData(); DataInput da = new DataInputStream(new ByteArrayInputStream(bytes)); // Try to re-create the BlockLocation the same way as is done during // deserialization BlockLocation bl2 = new BlockLocation(); try { bl2.readFields(da); } catch (IOException e) { fail("Unable to deserialize BlockLocation: " + e.getMessage()); } // Check that we got back what we started with verifyDeserialization(bl2.getHosts(), hosts); verifyDeserialization(bl2.getNames(), names); verifyDeserialization(bl2.getTopologyPaths(), topologyPaths); assertEquals(bl2.getOffset(), offset); assertEquals(bl2.getLength(), length);
5228 -1009652481apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc NOTE: this logic should be better, mimics previous implementation NOTE: this logic should be better, mimics previous implementation FILE_PATH_CHANGED registerCommands(CommandFactory) public static void registerCommands(CommandFactory factory) factory.addClass(Merge.class, "-getmerge"); factory.addClass(Cp.class, "-cp"); factory.addClass(CopyFromLocal.class, "-copyFromLocal"); factory.addClass(CopyToLocal.class, "-copyToLocal"); factory.addClass(Get.class, "-get"); factory.addClass(Put.class, "-put");
5227 -1009652482apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc too bad we can't tell exactly why it failed... too bad we can't tell exactly why it failed... FILE_PATH_CHANGED registerCommands(CommandFactory) public static void registerCommands(CommandFactory factory) factory.addClass(Merge.class, "-getmerge"); factory.addClass(Cp.class, "-cp"); factory.addClass(CopyFromLocal.class, "-copyFromLocal"); factory.addClass(CopyToLocal.class, "-copyToLocal"); factory.addClass(Get.class, "-get"); factory.addClass(Put.class, "-put");
5226 -1009652483apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc TODO: this really should be a -nl option TODO: this really should be a -nl option FILE_PATH_CHANGED registerCommands(CommandFactory) public static void registerCommands(CommandFactory factory) factory.addClass(Merge.class, "-getmerge"); factory.addClass(Cp.class, "-cp"); factory.addClass(CopyFromLocal.class, "-copyFromLocal"); factory.addClass(CopyToLocal.class, "-copyToLocal"); factory.addClass(Get.class, "-get"); factory.addClass(Put.class, "-put");
5225 -1009652526apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc The following XDR recipe was done through a careful reading of gm_protocol.x in Ganglia 3.1 and carefully examining the output of the gmetric utility with strace. The following XDR recipe was done through a careful reading of gm_protocol.x in Ganglia 3.1 and carefully examining the output of the gmetric utility with strace. FILE_PATH_CHANGED emitMetric(String, String, String) protected void emitMetric(String name, String type, String value) throws IOException if (name == null) { LOG.warn("Metric was emitted with no name."); return; } else if (value == null) { LOG.warn("Metric name " + name + " was emitted with a null value."); return; } else if (type == null) { LOG.warn("Metric name " + name + ", value " + value + " has no type."); return; } LOG.debug("Emitting metric " + name + ", type " + type + ", value " + value + " from hostname" + hostName); String units = getUnits(name); if (units == null) { LOG.warn("Metric name " + name + ", value " + value + " had 'null' units"); units = ""; } int slope = getSlope(name); int tmax = getTmax(name); int dmax = getDmax(name); offset = 0; String groupName = name.substring(0, name.lastIndexOf(".")); // The following XDR recipe was done through a careful reading of // gm_protocol.x in Ganglia 3.1 and carefully examining the output of // the gmetric utility with strace. // First we send out a metadata message // metric_id = metadata_msg xdr_int(128); // hostname xdr_string(hostName); // metric name xdr_string(name); // spoof = False xdr_int(0); // metric type xdr_string(type); // metric name xdr_string(name); // units xdr_string(units); // slope xdr_int(slope); // tmax, the maximum time between metrics xdr_int(tmax); // dmax, the maximum data value xdr_int(dmax); xdr_int(1); /*Num of the entries in extra_value field for Ganglia 3.1.x*/ xdr_string("GROUP"); /*Group attribute*/ xdr_string(groupName); /*Group value*/ for (SocketAddress socketAddress : metricsServers) { DatagramPacket packet = new DatagramPacket(buffer, offset, socketAddress); datagramSocket.send(packet); } // Now we send out a message with the actual value. // Technically, we only need to send out the metadata message once for // each metric, but I don't want to have to record which metrics we did and // did not send. offset = 0; // we are sending a string value xdr_int(133); // hostName xdr_string(hostName); // metric name xdr_string(name); // spoof = False xdr_int(0); // format field xdr_string("%s"); // metric value xdr_string(value); for (SocketAddress socketAddress : metricsServers) { DatagramPacket packet = new DatagramPacket(buffer, offset, socketAddress); datagramSocket.send(packet); }
5224 -1009652599apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc maybe too long maybe too long FILE_PATH_CHANGED writeString(DataOutput, String) public static int writeString(DataOutput out, String s) throws IOException if (s.length() > 0xffff / 3) { // maybe too long LOG.warn("truncating long string: " + s.length() + " chars, starting with " + s.substring(0, 20)); s = s.substring(0, 0xffff / 3); } int len = utf8Length(s); if (// double-check length len > 0xffff) throw new IOException("string too long!"); out.writeShort(len); writeChars(out, s, 0, s.length()); return len;
5223 -1009652600apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc maybe too long maybe too long FILE_PATH_CHANGED set(String) public void set(String string) if (string.length() > 0xffff / 3) { // maybe too long LOG.warn("truncating long string: " + string.length() + " chars, starting with " + string.substring(0, 20)); string = string.substring(0, 0xffff / 3); } // compute length length = utf8Length(string); if (// double-check length length > 0xffff) throw new RuntimeException("string too long!"); if (// grow buffer bytes == null || length > bytes.length) bytes = new byte[length]; try { // avoid sync'd allocations DataOutputBuffer obuf = OBUF_FACTORY.get(); obuf.reset(); writeChars(obuf, string, 0, string.length()); System.arraycopy(obuf.getData(), 0, bytes, 0, length); } catch (IOException e) { throw new RuntimeException(e); }
5222 -1009652546apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc Set new default probability if specified through a system.property If neither is specified set default probability to DEFAULT_PROB Set new default probability if specified through a system.property If neither is specified set default probability to DEFAULT_PROB FILE_PATH_CHANGED injectCriteria(String) public static boolean injectCriteria(String klassName) boolean trigger = false; if (generator.nextFloat() < getProbability(klassName)) { trigger = true; } return trigger;
5220 -1009652544apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc Some filesystem like HDFS ignore the \"x\" bit if the permission. Others like localFs does not. Override the method below if the file system being tested masks our certain bits for file masks. Some filesystem like HDFS ignore the \"x\" bit if the permission. Others like localFs does not. Override the method below if the file system being tested masks our certain bits for file masks. FILE_PATH_CHANGED setUp() public void setUp() throws Exception fc.mkdir(getTestRootPath(fc), FileContext.DEFAULT_PERM, true);
5219 -1009652578apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc Send it to S3 TODO: Use passed in Progressable to report progress. Send it to S3 TODO: Use passed in Progressable to report progress. FILE_PATH_CHANGED endBlock() private synchronized void endBlock() throws IOException // // Done with local copy // backupStream.close(); // // Send it to S3 // // TODO: Use passed in Progressable to report progress. nextBlockOutputStream(); store.storeBlock(nextBlock, backupFile); internalClose(); // // Delete local backup, start new one // boolean b = backupFile.delete(); if (!b) { LOG.warn("Ignoring failed delete"); } backupFile = newBackupFile(); backupStream = new FileOutputStream(backupFile); bytesWrittenToBlock = 0;
5218 -1009652585apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc move checksum file too move checksum file too FILE_PATH_CHANGED reportChecksumFailure(Path, FSDataInputStream, long, FSDataInputStream, long) public boolean reportChecksumFailure(Path p, FSDataInputStream in, long inPos, FSDataInputStream sums, long sumsPos) try { // canonicalize f File f = ((RawLocalFileSystem) fs).pathToFile(p).getCanonicalFile(); // find highest writable parent dir of f on the same device String device = new DF(f, getConf()).getMount(); File parent = f.getParentFile(); File dir = null; while (parent != null && parent.canWrite() && parent.toString().startsWith(device)) { dir = parent; parent = parent.getParentFile(); } if (dir == null) { throw new IOException("not able to find the highest writable parent dir"); } // move the file there File badDir = new File(dir, "bad_files"); if (!badDir.mkdirs()) { if (!badDir.isDirectory()) { throw new IOException("Mkdirs failed to create " + badDir.toString()); } } String suffix = "." + rand.nextInt(); File badFile = new File(badDir, f.getName() + suffix); LOG.warn("Moving bad file " + f + " to " + badFile); // close it first in.close(); // rename it boolean b = f.renameTo(badFile); if (!b) { LOG.warn("Ignoring failure of renameTo"); } // move checksum file too File checkFile = ((RawLocalFileSystem) fs).pathToFile(getChecksumFile(p)); b = checkFile.renameTo(new File(badDir, checkFile.getName() + suffix)); if (!b) { LOG.warn("Ignoring failure of renameTo"); } } catch (IOException e) { LOG.warn("Error moving bad file " + p + ": " + e); } return false;
5217 -1009652522apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc better safe than sorry (should never happen) better safe than sorry (should never happen) FILE_PATH_CHANGED cacheGroupsAdd(List) public void cacheGroupsAdd(List groups) throws IOException for (String group : groups) { if (group.length() == 0) { // better safe than sorry (should never happen) } else if (group.charAt(0) == '@') { if (!NetgroupCache.isCached(group)) { NetgroupCache.add(group, getUsersForNetgroup(group)); } } else { // unix group, not caching } }
5216 -1009652553apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc Verify that skip trash option really skips the trash for rmr Verify that skip trash option really skips the trash for rmr FILE_PATH_CHANGED writeFile(FileSystem, Path) protected static Path writeFile(FileSystem fs, Path f) throws IOException DataOutputStream out = fs.create(f); out.writeBytes("dhruba: " + f); out.close(); assertTrue(fs.exists(f)); return f;
5215 -1009652554apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc Verify skip trash option really works Verify skip trash option really works FILE_PATH_CHANGED writeFile(FileSystem, Path) protected static Path writeFile(FileSystem fs, Path f) throws IOException DataOutputStream out = fs.create(f); out.writeBytes("dhruba: " + f); out.close(); assertTrue(fs.exists(f)); return f;
5214 -1009652561apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc verify that after expunging the Trash, it really goes away verify that after expunging the Trash, it really goes away FILE_PATH_CHANGED writeFile(FileSystem, Path) protected static Path writeFile(FileSystem fs, Path f) throws IOException DataOutputStream out = fs.create(f); out.writeBytes("dhruba: " + f); out.close(); assertTrue(fs.exists(f)); return f;
5212 -1009652535apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc perhaps a VIP is failing over perhaps a VIP is failing over FILE_PATH_CHANGED waitForProtocolProxy(Class, long, InetSocketAddress, Configuration, int, long) public static ProtocolProxy waitForProtocolProxy(Class protocol, long clientVersion, InetSocketAddress addr, Configuration conf, int rpcTimeout, long timeout) throws IOException long startTime = System.currentTimeMillis(); IOException ioe; while (true) { try { return getProtocolProxy(protocol, clientVersion, addr, UserGroupInformation.getCurrentUser(), conf, NetUtils.getDefaultSocketFactory(conf), rpcTimeout); } catch (ConnectException se) { // namenode has not been started LOG.info("Server at " + addr + " not available yet, Zzzzz..."); ioe = se; } catch (SocketTimeoutException te) { // namenode is busy LOG.info("Problem connecting to server: " + addr); ioe = te; } catch (NoRouteToHostException nrthe) { // perhaps a VIP is failing over LOG.info("No route to host for server: " + addr); ioe = nrthe; } // check if timed out if (System.currentTimeMillis() - timeout >= startTime) { throw ioe; } // wait for retry try { Thread.sleep(1000); } catch (InterruptedException ie) { // IGNORE } }
5210 -1009652506apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc implicitly recurse once for cmdline directories implicitly recurse once for cmdline directories FILE_PATH_CHANGED registerCommands(CommandFactory) public static void registerCommands(CommandFactory factory) factory.addClass(Ls.class, "-ls"); factory.addClass(Lsr.class, "-lsr");
5209 -1009652555apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc TODO: remember the longest key in a TFile, and use it to replace MAX_KEY_SIZE. TODO: remember the longest key in a TFile, and use it to replace MAX_KEY_SIZE. FILE_PATH_CHANGED getChunkBufferSize(Configuration) static int getChunkBufferSize(Configuration conf) int ret = conf.getInt(CHUNK_BUF_SIZE_ATTR, 1024 * 1024); return (ret > 0) ? ret : 1024 * 1024;
5208 -1009652556apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc flag to ensure value is only examined once. flag to ensure value is only examined once. FILE_PATH_CHANGED getChunkBufferSize(Configuration) static int getChunkBufferSize(Configuration conf) int ret = conf.getInt(CHUNK_BUF_SIZE_ATTR, 1024 * 1024); return (ret > 0) ? ret : 1024 * 1024;
5207 -1009652501apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc this is very ugly, but needed to avoid breaking hdfs tests... if a path has no authority, then the FileStatus from globStatus will add the \"-fs\" authority into the path, so we need to sub it back out to satisfy the tests this is very ugly, but needed to avoid breaking hdfs tests... if a path has no authority, then the FileStatus from globStatus will add the \"-fs\" authority into the path, so we need to sub it back out to satisfy the tests FILE_PATH_CHANGED expandAsGlob(String, Configuration) public static PathData[] expandAsGlob(String pattern, Configuration conf) throws IOException Path globPath = new Path(pattern); FileSystem fs = globPath.getFileSystem(conf); FileStatus[] stats = fs.globStatus(globPath); PathData[] items = null; if (stats == null) { // not a glob & file not found, so add the path with a null stat items = new PathData[] { new PathData(fs, pattern, null) }; } else if (// this is very ugly, but needed to avoid breaking hdfs tests... // if a path has no authority, then the FileStatus from globStatus // will add the "-fs" authority into the path, so we need to sub // it back out to satisfy the tests stats.length == 1 && stats[0].getPath().equals(fs.makeQualified(globPath))) { // if the fq path is identical to the pattern passed, use the pattern // to initialize the string value items = new PathData[] { new PathData(fs, pattern, stats[0]) }; } else { // convert stats to PathData items = new PathData[stats.length]; int i = 0; for (FileStatus stat : stats) { items[i++] = new PathData(fs, stat); } } return items;
5206 -1009652479apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc Ignored the mbean itself was not found, which should never happen because we just accessed it (perhaps something unregistered in-between) but if this happens just don't output the attribute. Ignored the mbean itself was not found, which should never happen because we just accessed it (perhaps something unregistered in-between) but if this happens just don't output the attribute. FILE_PATH_CHANGED writeAttribute(JsonGenerator, ObjectName, MBeanAttributeInfo) private void writeAttribute(JsonGenerator jg, ObjectName oname, MBeanAttributeInfo attr) throws IOException if (!attr.isReadable()) { return; } String attName = attr.getName(); if ("modelerType".equals(attName)) { return; } if (attName.indexOf("=") >= 0 || attName.indexOf(":") >= 0 || attName.indexOf(" ") >= 0) { return; } Object value = null; try { value = mBeanServer.getAttribute(oname, attName); } catch (AttributeNotFoundException e) { // Ignored the attribute was not found, which should never happen because the bean // just told us that it has this attribute, but if this happens just don't output // the attribute. return; } catch (MBeanException e) { // The code inside the attribute getter threw an exception so log it, and // skip outputting the attribute LOG.error("getting attribute " + attName + " of " + oname + " threw an exception", e); return; } catch (RuntimeException e) { // For some reason even with an MBeanException available to them Runtime exceptions // can still find their way through, so treat them the same as MBeanException LOG.error("getting attribute " + attName + " of " + oname + " threw an exception", e); return; } catch (ReflectionException e) { // This happens when the code inside the JMX bean (setter?? from the java docs) // threw an exception, so log it and skip outputting the attribute LOG.error("getting attribute " + attName + " of " + oname + " threw an exception", e); return; } catch (InstanceNotFoundException e) { // Ignored the mbean itself was not found, which should never happen because we // just accessed it (perhaps something unregistered in-between) but if this // happens just don't output the attribute. return; } writeAttribute(jg, attName, value);
5205 -1009652610apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc Ugly utility, maybe someone else can do this better Ugly utility, maybe someone else can do this better FILE_PATH_CHANGED readCompressedString(DataInput) public static String readCompressedString(DataInput in) throws IOException byte[] bytes = readCompressedByteArray(in); if (bytes == null) return null; return new String(bytes, "UTF-8");
5204 -1009652576apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc Following is brittle. Is there a better way? Following is brittle. Is there a better way? FILE_PATH_CHANGED retrieveMetadata(String) public FileMetadata retrieveMetadata(String key) throws IOException try { S3Object object = s3Service.getObjectDetails(bucket, key); return new FileMetadata(key, object.getContentLength(), object.getLastModifiedDate().getTime()); } catch (S3ServiceException e) { // Following is brittle. Is there a better way? if (e.getMessage().contains("ResponseCode=404")) { return null; } handleServiceException(e); // never returned - keep compiler happy return null; }
5203 -1009652577apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc walk through the list, searching. Not the most efficient way, but this in intended to be used rarely, so we keep it simple. As an optimization, we can keep a hashmap of record name to its RTI, for later. walk through the list, searching. Not the most efficient way, but this in intended to be used rarely, so we keep it simple. As an optimization, we can keep a hashmap of record name to its RTI, for later. FILE_PATH_CHANGED findStruct(String) // walk through the list, searching. Not the most efficient way, but this // in intended to be used rarely, so we keep it simple. // As an optimization, we can keep a hashmap of record name to its RTI, for later. for (FieldTypeInfo ti : typeInfos) { if ((0 == ti.getFieldID().compareTo(name)) && (ti.getTypeID().getTypeVal() == RIOType.STRUCT)) { return (StructTypeID) ti.getTypeID(); } } return null; StructTypeID findStruct(String name)
5202 -1009652579apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc Iterate up to N_ITERS times to improve the tables. Iterate up to N_ITERS times to improve the tables. FILE_PATH_CHANGED hbMakeCodeLengths(char[], int[], int, int) protected static void hbMakeCodeLengths(char[] len, int[] freq, int alphaSize, int maxLen) /* * Nodes and heap entries run from 1. Entry 0 for both the heap and * nodes is a sentinel. */ final int[] heap = new int[MAX_ALPHA_SIZE * 2]; final int[] weight = new int[MAX_ALPHA_SIZE * 2]; final int[] parent = new int[MAX_ALPHA_SIZE * 2]; for (int i = alphaSize; --i >= 0; ) { weight[i + 1] = (freq[i] == 0 ? 1 : freq[i]) << 8; } for (boolean tooLong = true; tooLong; ) { tooLong = false; int nNodes = alphaSize; int nHeap = 0; heap[0] = 0; weight[0] = 0; parent[0] = -2; for (int i = 1; i <= alphaSize; i++) { parent[i] = -1; nHeap++; heap[nHeap] = i; int zz = nHeap; int tmp = heap[zz]; while (weight[tmp] < weight[heap[zz >> 1]]) { heap[zz] = heap[zz >> 1]; zz >>= 1; } heap[zz] = tmp; } // assert (nHeap < (MAX_ALPHA_SIZE + 2)) : nHeap; while (nHeap > 1) { int n1 = heap[1]; heap[1] = heap[nHeap]; nHeap--; int yy = 0; int zz = 1; int tmp = heap[1]; while (true) { yy = zz << 1; if (yy > nHeap) { break; } if ((yy < nHeap) && (weight[heap[yy + 1]] < weight[heap[yy]])) { yy++; } if (weight[tmp] < weight[heap[yy]]) { break; } heap[zz] = heap[yy]; zz = yy; } heap[zz] = tmp; int n2 = heap[1]; heap[1] = heap[nHeap]; nHeap--; yy = 0; zz = 1; tmp = heap[1]; while (true) { yy = zz << 1; if (yy > nHeap) { break; } if ((yy < nHeap) && (weight[heap[yy + 1]] < weight[heap[yy]])) { yy++; } if (weight[tmp] < weight[heap[yy]]) { break; } heap[zz] = heap[yy]; zz = yy; } heap[zz] = tmp; nNodes++; parent[n1] = parent[n2] = nNodes; final int weight_n1 = weight[n1]; final int weight_n2 = weight[n2]; weight[nNodes] = (((weight_n1 & 0xffffff00) + (weight_n2 & 0xffffff00)) | (1 + (((weight_n1 & 0x000000ff) > (weight_n2 & 0x000000ff)) ? (weight_n1 & 0x000000ff) : (weight_n2 & 0x000000ff)))); parent[nNodes] = -1; nHeap++; heap[nHeap] = nNodes; tmp = 0; zz = nHeap; tmp = heap[zz]; final int weight_tmp = weight[tmp]; while (weight_tmp < weight[heap[zz >> 1]]) { heap[zz] = heap[zz >> 1]; zz >>= 1; } heap[zz] = tmp; } // assert (nNodes < (MAX_ALPHA_SIZE * 2)) : nNodes; for (int i = 1; i <= alphaSize; i++) { int j = 0; int k = i; for (int parent_k; (parent_k = parent[k]) >= 0; ) { k = parent_k; j++; } len[i - 1] = (char) j; if (j > maxLen) { tooLong = true; } } if (tooLong) { for (int i = 1; i < alphaSize; i++) { int j = weight[i] >> 8; j = 1 + (j >> 1); weight[i] = j << 8; } } }
5201 -1009652580apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc Now another magic 48-bit number, 0x177245385090, to indicate the end of the last block. (sqrt(pi), if you want to know. I did want to use e, but it contains too much repetition -- 27 18 28 18 28 46 -- for me to feel statistically comfortable. Call me paranoid.) Now another magic 48-bit number, 0x177245385090, to indicate the end of the last block. (sqrt(pi), if you want to know. I did want to use e, but it contains too much repetition -- 27 18 28 18 28 46 -- for me to feel statistically comfortable. Call me paranoid.) FILE_PATH_CHANGED hbMakeCodeLengths(char[], int[], int, int) protected static void hbMakeCodeLengths(char[] len, int[] freq, int alphaSize, int maxLen) /* * Nodes and heap entries run from 1. Entry 0 for both the heap and * nodes is a sentinel. */ final int[] heap = new int[MAX_ALPHA_SIZE * 2]; final int[] weight = new int[MAX_ALPHA_SIZE * 2]; final int[] parent = new int[MAX_ALPHA_SIZE * 2]; for (int i = alphaSize; --i >= 0; ) { weight[i + 1] = (freq[i] == 0 ? 1 : freq[i]) << 8; } for (boolean tooLong = true; tooLong; ) { tooLong = false; int nNodes = alphaSize; int nHeap = 0; heap[0] = 0; weight[0] = 0; parent[0] = -2; for (int i = 1; i <= alphaSize; i++) { parent[i] = -1; nHeap++; heap[nHeap] = i; int zz = nHeap; int tmp = heap[zz]; while (weight[tmp] < weight[heap[zz >> 1]]) { heap[zz] = heap[zz >> 1]; zz >>= 1; } heap[zz] = tmp; } // assert (nHeap < (MAX_ALPHA_SIZE + 2)) : nHeap; while (nHeap > 1) { int n1 = heap[1]; heap[1] = heap[nHeap]; nHeap--; int yy = 0; int zz = 1; int tmp = heap[1]; while (true) { yy = zz << 1; if (yy > nHeap) { break; } if ((yy < nHeap) && (weight[heap[yy + 1]] < weight[heap[yy]])) { yy++; } if (weight[tmp] < weight[heap[yy]]) { break; } heap[zz] = heap[yy]; zz = yy; } heap[zz] = tmp; int n2 = heap[1]; heap[1] = heap[nHeap]; nHeap--; yy = 0; zz = 1; tmp = heap[1]; while (true) { yy = zz << 1; if (yy > nHeap) { break; } if ((yy < nHeap) && (weight[heap[yy + 1]] < weight[heap[yy]])) { yy++; } if (weight[tmp] < weight[heap[yy]]) { break; } heap[zz] = heap[yy]; zz = yy; } heap[zz] = tmp; nNodes++; parent[n1] = parent[n2] = nNodes; final int weight_n1 = weight[n1]; final int weight_n2 = weight[n2]; weight[nNodes] = (((weight_n1 & 0xffffff00) + (weight_n2 & 0xffffff00)) | (1 + (((weight_n1 & 0x000000ff) > (weight_n2 & 0x000000ff)) ? (weight_n1 & 0x000000ff) : (weight_n2 & 0x000000ff)))); parent[nNodes] = -1; nHeap++; heap[nHeap] = nNodes; tmp = 0; zz = nHeap; tmp = heap[zz]; final int weight_tmp = weight[tmp]; while (weight_tmp < weight[heap[zz >> 1]]) { heap[zz] = heap[zz >> 1]; zz >>= 1; } heap[zz] = tmp; } // assert (nNodes < (MAX_ALPHA_SIZE * 2)) : nNodes; for (int i = 1; i <= alphaSize; i++) { int j = 0; int k = i; for (int parent_k; (parent_k = parent[k]) >= 0; ) { k = parent_k; j++; } len[i - 1] = (char) j; if (j > maxLen) { tooLong = true; } } if (tooLong) { for (int i = 1; i < alphaSize; i++) { int j = weight[i] >> 8; j = 1 + (j >> 1); weight[i] = j << 8; } } }
5199 -1009652545apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc Default impl assumes that permissions do not matter calling the regular create is good enough. FSs that implement permissions should override this. Default impl assumes that permissions do not matter calling the regular create is good enough. FSs that implement permissions should override this. FILE_PATH_CHANGED createInternal(Path, EnumSet, FsPermission, int, short, long, Progressable, int, boolean) public FSDataOutputStream createInternal(Path f, EnumSet flag, FsPermission absolutePermission, int bufferSize, short replication, long blockSize, Progressable progress, int bytesPerChecksum, boolean createParent) throws IOException checkPath(f); // Default impl assumes that permissions do not matter // calling the regular create is good enough. // FSs that implement permissions should override this. if (!createParent) { // parent must exist. // since this.create makes parent dirs automatically // we must throw exception if parent does not exist. final FileStatus stat = getFileStatus(f.getParent()); if (stat == null) { throw new FileNotFoundException("Missing parent:" + f); } if (!stat.isDirectory()) { throw new ParentNotDirectoryException("parent is not a dir:" + f); } // parent does exist - go ahead with create of file. } return fsImpl.primitiveCreate(f, absolutePermission, flag, bufferSize, replication, blockSize, progress, bytesPerChecksum);
5197 -1009652589apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc Using default block size since there is no way in FTP client to know of block sizes on server. The assumption could be less than ideal. Using default block size since there is no way in FTP client to know of block sizes on server. The assumption could be less than ideal. FILE_PATH_CHANGED getFileStatus(FTPFile, Path) private FileStatus getFileStatus(FTPFile ftpFile, Path parentPath) long length = ftpFile.getSize(); boolean isDir = ftpFile.isDirectory(); int blockReplication = 1; // Using default block size since there is no way in FTP client to know of // block sizes on server. The assumption could be less than ideal. long blockSize = DEFAULT_BLOCK_SIZE; long modTime = ftpFile.getTimestamp().getTimeInMillis(); long accessTime = 0; FsPermission permission = getPermissions(ftpFile); String user = ftpFile.getUser(); String group = ftpFile.getGroup(); Path filePath = new Path(parentPath, ftpFile.getName()); return new FileStatus(length, isDir, blockReplication, blockSize, modTime, accessTime, permission, user, group, filePath.makeQualified(this));
5196 -1009652562apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc Check that every permission has its sticky bit represented correctly Check that every permission has its sticky bit represented correctly FILE_PATH_CHANGED testStickyBitToString() public void testStickyBitToString() // Check that every permission has its sticky bit represented correctly for (boolean sb : new boolean[] { false, true }) { for (FsAction u : FsAction.values()) { for (FsAction g : FsAction.values()) { for (FsAction o : FsAction.values()) { FsPermission f = new FsPermission(u, g, o, sb); if (f.getStickyBit() && f.getOtherAction().implies(EXECUTE)) assertEquals('t', f.toString().charAt(8)); else if (f.getStickyBit() && !f.getOtherAction().implies(EXECUTE)) assertEquals('T', f.toString().charAt(8)); else if (!f.getStickyBit() && f.getOtherAction().implies(EXECUTE)) assertEquals('x', f.toString().charAt(8)); else assertEquals('-', f.toString().charAt(8)); } } } }
5195 -1009652558apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc move the cursor to the beginning of the tail, containing: offset to the meta block index, version and magic move the cursor to the beginning of the tail, containing: offset to the meta block index, version and magic FILE_PATH_CHANGED register(long, long, long) public void register(long raw, long offsetStart, long offsetEnd) mp
5194 -1009652523apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc better safe than sorry (should never happen) better safe than sorry (should never happen) FILE_PATH_CHANGED cacheGroupsAdd(List) public void cacheGroupsAdd(List groups) throws IOException for (String group : groups) { if (group.length() == 0) { // better safe than sorry (should never happen) } else if (group.charAt(0) == '@') { if (!NetgroupCache.isCached(group)) { NetgroupCache.add(group, getUsersForNetgroup(group)); } } else { // unix group, not caching } }
5193 -1009652533apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc XXX Twenty internal version does not support --script option. XXX Twenty internal version does not support --script option. FILE_PATH_CHANGED getCommand(String, String) protected String[] getCommand(String command, String confDir) ArrayList cmdArgs = new ArrayList(); File binDir = getBinDir(); cmdArgs.add("ssh"); cmdArgs.add(hostName); cmdArgs.add(binDir.getAbsolutePath() + File.separator + SCRIPT_NAME); cmdArgs.add("--config"); cmdArgs.add(confDir); // XXX Twenty internal version does not support --script option. cmdArgs.add(command); cmdArgs.add(daemonName); return (String[]) cmdArgs.toArray(new String[cmdArgs.size()]);
5192 -1009652496apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc Test resolvePath(p) TODO In the tests below replace fcView.getDefaultFileSystem().resolvePath() fcView.resolvePath() Test resolvePath(p) TODO In the tests below replace fcView.getDefaultFileSystem().resolvePath() fcView.resolvePath() FILE_PATH_CHANGED testResolvePathInternalPaths() public void testResolvePathInternalPaths() throws IOException Assert.assertEquals(new Path("/"), fcView.resolvePath(new Path("/"))); Assert.assertEquals(new Path("/internalDir"), fcView.resolvePath(new Path("/internalDir")));
5191 -1009652485apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc TODO: should probably allow path arguments for the filesystems TODO: should probably allow path arguments for the filesystems FILE_PATH_CHANGED processOptions(LinkedList) protected void processOptions(LinkedList args) throws IOException CommandFormat cf = new CommandFormat(0, 0); cf.parse(args);
5190 -1009652486apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc TODO: if the user wants the trash to be used but there is any problem (ie. creating the trash dir, moving the item to be deleted, etc), then the path will just be deleted because moveToTrash returns false and it falls thru to fs.delete. this doesn't seem right TODO: if the user wants the trash to be used but there is any problem (ie. creating the trash dir, moving the item to be deleted, etc), then the path will just be deleted because moveToTrash returns false and it falls thru to fs.delete. this doesn't seem right FILE_PATH_CHANGED registerCommands(CommandFactory) public static void registerCommands(CommandFactory factory) factory.addClass(Rm.class, "-rm"); factory.addClass(Rmr.class, "-rmr"); factory.addClass(Expunge.class, "-expunge");
5189 -1009652472apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc Doing a second call with faults disabled should return fine -- ie the internal state of the client or server should not be broken by the failed call Doing a second call with faults disabled should return fine -- ie the internal state of the client or server should not be broken by the failed call FILE_PATH_CHANGED call(Class, Writable, long) public Writable call(Class protocol, Writable param, long receiveTime) throws IOException if (sleep) { // sleep a bit try { Thread.sleep(RANDOM.nextInt(PING_INTERVAL) + MIN_SLEEP_TIME); } catch (InterruptedException e) { } } if (responseClass != null) { try { return responseClass.newInstance(); } catch (Exception e) { throw new RuntimeException(e); } } else { // echo param as result return param; }
5188 -1009652517apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc Read the int[] object as written by ObjectWritable, but \"going around\" ObjectWritable Read the int[] object as written by ObjectWritable, but \"going around\" ObjectWritable FILE_PATH_CHANGED testOldFormat() public void testOldFormat() throws IOException // Make sure we still correctly write the old format if desired. // Write the data array with old ObjectWritable API // which will set allowCompactArrays false. ObjectWritable.writeObject(out, i, i.getClass(), null); // Get ready to read it back in.reset(out.getData(), out.getLength()); // Read the int[] object as written by ObjectWritable, but // "going around" ObjectWritable @SuppressWarnings("deprecation") String className = UTF8.readString(in); assertEquals("The int[] written by ObjectWritable as a non-compact array " + "was not labelled as an array of int", i.getClass().getName(), className); int length = in.readInt(); assertEquals("The int[] written by ObjectWritable as a non-compact array " + "was not expected length", i.length, length); int[] readValue = new int[length]; try { for (int i = 0; i < length; i++) { readValue[i] = (int) ((Integer) ObjectWritable.readObject(in, null)); } } catch (Exception e) { fail("The int[] written by ObjectWritable as a non-compact array " + "was corrupted. Failed to correctly read int[] of length " + length + ". Got exception:\n" + StringUtils.stringifyException(e)); } assertTrue("The int[] written by ObjectWritable as a non-compact array " + "was corrupted.", Arrays.equals(i, readValue));
5187 -1009652518apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc Read the APW object as written by ObjectWritable, but \"going around\" ObjectWritable Read the APW object as written by ObjectWritable, but \"going around\" ObjectWritable FILE_PATH_CHANGED testObjectLabeling() public void testObjectLabeling() throws IOException // Do a few tricky experiments to make sure things are being written // the way we expect // Write the data array with ObjectWritable // which will indirectly write it using APW.Internal ObjectWritable.writeObject(out, i, i.getClass(), null, true); // Write the corresponding APW directly with ObjectWritable ArrayPrimitiveWritable apw = new ArrayPrimitiveWritable(i); ObjectWritable.writeObject(out, apw, apw.getClass(), null, true); // Get ready to read it back in.reset(out.getData(), out.getLength()); // Read the int[] object as written by ObjectWritable, but // "going around" ObjectWritable String className = UTF8.readString(in); assertEquals("The int[] written by ObjectWritable was not labelled as " + "an ArrayPrimitiveWritable.Internal", ArrayPrimitiveWritable.Internal.class.getName(), className); ArrayPrimitiveWritable.Internal apwi = new ArrayPrimitiveWritable.Internal(); apwi.readFields(in); assertEquals("The ArrayPrimitiveWritable.Internal component type was corrupted", int.class, apw.getComponentType()); assertTrue("The int[] written by ObjectWritable as " + "ArrayPrimitiveWritable.Internal was corrupted", Arrays.equals(i, (int[]) (apwi.get()))); // Read the APW object as written by ObjectWritable, but // "going around" ObjectWritable String declaredClassName = UTF8.readString(in); assertEquals("The APW written by ObjectWritable was not labelled as " + "declaredClass ArrayPrimitiveWritable", ArrayPrimitiveWritable.class.getName(), declaredClassName); className = UTF8.readString(in); assertEquals("The APW written by ObjectWritable was not labelled as " + "class ArrayPrimitiveWritable", ArrayPrimitiveWritable.class.getName(), className); ArrayPrimitiveWritable apw2 = new ArrayPrimitiveWritable(); apw2.readFields(in); assertEquals("The ArrayPrimitiveWritable component type was corrupted", int.class, apw2.getComponentType()); assertTrue("The int[] written by ObjectWritable as " + "ArrayPrimitiveWritable was corrupted", Arrays.equals(i, (int[]) (apw2.get())));
5186 -1009652519apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc Read the int[] object as written by ObjectWritable, but \"going around\" ObjectWritable Read the int[] object as written by ObjectWritable, but \"going around\" ObjectWritable FILE_PATH_CHANGED testObjectLabeling() public void testObjectLabeling() throws IOException // Do a few tricky experiments to make sure things are being written // the way we expect // Write the data array with ObjectWritable // which will indirectly write it using APW.Internal ObjectWritable.writeObject(out, i, i.getClass(), null, true); // Write the corresponding APW directly with ObjectWritable ArrayPrimitiveWritable apw = new ArrayPrimitiveWritable(i); ObjectWritable.writeObject(out, apw, apw.getClass(), null, true); // Get ready to read it back in.reset(out.getData(), out.getLength()); // Read the int[] object as written by ObjectWritable, but // "going around" ObjectWritable String className = UTF8.readString(in); assertEquals("The int[] written by ObjectWritable was not labelled as " + "an ArrayPrimitiveWritable.Internal", ArrayPrimitiveWritable.Internal.class.getName(), className); ArrayPrimitiveWritable.Internal apwi = new ArrayPrimitiveWritable.Internal(); apwi.readFields(in); assertEquals("The ArrayPrimitiveWritable.Internal component type was corrupted", int.class, apw.getComponentType()); assertTrue("The int[] written by ObjectWritable as " + "ArrayPrimitiveWritable.Internal was corrupted", Arrays.equals(i, (int[]) (apwi.get()))); // Read the APW object as written by ObjectWritable, but // "going around" ObjectWritable String declaredClassName = UTF8.readString(in); assertEquals("The APW written by ObjectWritable was not labelled as " + "declaredClass ArrayPrimitiveWritable", ArrayPrimitiveWritable.class.getName(), declaredClassName); className = UTF8.readString(in); assertEquals("The APW written by ObjectWritable was not labelled as " + "class ArrayPrimitiveWritable", ArrayPrimitiveWritable.class.getName(), className); ArrayPrimitiveWritable apw2 = new ArrayPrimitiveWritable(); apw2.readFields(in); assertEquals("The ArrayPrimitiveWritable component type was corrupted", int.class, apw2.getComponentType()); assertTrue("The int[] written by ObjectWritable as " + "ArrayPrimitiveWritable was corrupted", Arrays.equals(i, (int[]) (apw2.get())));
5185 -1009652529apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc NOTE: this is the one place we do NOT want to save the number of bytes sent (nRemaining here) into lastBytesSent: since we are resending what we've already sent before, offset is nonzero in general (only way it could be zero is if it already equals nRemaining), which would then screw up the offset calculation _next_ time around. IOW, getRemaining() is in terms of the original, zero-offset bufferload, so lastBytesSent must be as well. Cheesy ASCII art: <------------ m, lastBytesSent -----------> +===============================================+ buffer: |1111111111|22222222222222222|333333333333| | +===============================================+ #1: <-- off -->|<-------- nRemaining ---------> #2: <----------- off ----------->|<-- nRem. --> NOTE: this is the one place we do NOT want to save the number of bytes sent (nRemaining here) into lastBytesSent: since we are resending what we've already sent before, offset is nonzero in general (only way it could be zero is if it already equals nRemaining), which would then screw up the offset calculation _next_ time around. IOW, getRemaining() is in terms of the original, zero-offset bufferload, so lastBytesSent must be as well. Cheesy ASCII art: <------------ m, lastBytesSent -----------> +===============================================+ buffer: |1111111111|22222222222222222|333333333333| | +===============================================+ #1: <-- off -->|<-------- nRemaining ---------> #2: <----------- off ----------->|<-- nRem. --> FILE_PATH_CHANGED read() public int read() throws IOException checkStream(); return (read(oneByte, 0, oneByte.length) == -1) ? -1 : (oneByte[0] & 0xff);
5184 -1009652473apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc an artificial way to make a key unique an artificial way to make a key unique FILE_PATH_CHANGED addFileSystemForTesting(URI, Configuration, FileSystem) static void addFileSystemForTesting(URI uri, Configuration conf, FileSystem fs) throws IOException CACHE.map.put(new Cache.Key(uri, conf), fs);
5183 -1009652474apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc Default impl is to assume that permissions do not matter and hence calling the regular mkdirs is good enough. FSs that implement permissions should override this. Default impl is to assume that permissions do not matter and hence calling the regular mkdirs is good enough. FSs that implement permissions should override this. FILE_PATH_CHANGED addFileSystemForTesting(URI, Configuration, FileSystem) static void addFileSystemForTesting(URI uri, Configuration conf, FileSystem fs) throws IOException CACHE.map.put(new Cache.Key(uri, conf), fs);
5182 -1009652475apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc Default impl assumes that permissions do not matter and nor does the bytesPerChecksum hence calling the regular create is good enough. FSs that implement permissions should override this. Default impl assumes that permissions do not matter and nor does the bytesPerChecksum hence calling the regular create is good enough. FSs that implement permissions should override this. FILE_PATH_CHANGED addFileSystemForTesting(URI, Configuration, FileSystem) static void addFileSystemForTesting(URI uri, Configuration conf, FileSystem fs) throws IOException CACHE.map.put(new Cache.Key(uri, conf), fs);
5181 -1009652495apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc hack to avoid most of the \"innocuous\" races. hack to avoid most of the \"innocuous\" races. FILE_PATH_CHANGED start() void start() if (startMBeans) startMBeans();
5180 -1009652582apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc give up give up FILE_PATH_CHANGED sortInternal(IndexedSortable, int, int, Progressable, int) private static void sortInternal(final IndexedSortable s, int p, int r, final Progressable rep, int depth) if (null != rep) { rep.progress(); } while (true) { if (r - p < 13) { for (int i = p; i < r; ++i) { for (int j = i; j > p && s.compare(j - 1, j) > 0; --j) { s.swap(j, j - 1); } } return; } if (--depth < 0) { // give up alt.sort(s, p, r, rep); return; } // select, move pivot into first position fix(s, (p + r) >>> 1, p); fix(s, (p + r) >>> 1, r - 1); fix(s, p, r - 1); // Divide int i = p; int j = r; int ll = p; int rr = r; int cr; while (true) { while (++i < j) { if ((cr = s.compare(i, p)) > 0) break; if (0 == cr && ++ll != i) { s.swap(ll, i); } } while (--j > i) { if ((cr = s.compare(p, j)) > 0) break; if (0 == cr && --rr != j) { s.swap(rr, j); } } if (i < j) s.swap(i, j); else break; } j = i; // swap pivot- and all eq values- into position while (ll >= p) { s.swap(ll--, --i); } while (rr < r) { s.swap(rr++, j++); } // Conquer // Recurse on smaller interval first to keep stack shallow assert i != j; if (i - p < r - j) { sortInternal(s, p, i, rep, depth); p = j; } else { sortInternal(s, j, r, rep, depth); r = i; } }
5179 -1009652527apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc Even though tmpDir is dangling symlink to tmp, fullyDelete(tmpDir) should delete tmpDir properly. Even though tmpDir is dangling symlink to tmp, fullyDelete(tmpDir) should delete tmpDir properly. FILE_PATH_CHANGED setupDirs() private void setupDirs() throws IOException Assert.assertFalse(del.exists()); Assert.assertFalse(tmp.exists()); Assert.assertFalse(partitioned.exists()); del.mkdirs(); tmp.mkdirs(); partitioned.mkdirs(); new File(del, FILE).createNewFile(); File tmpFile = new File(tmp, FILE); tmpFile.createNewFile(); // create directories dir1.mkdirs(); dir2.mkdirs(); new File(dir1, FILE).createNewFile(); new File(dir2, FILE).createNewFile(); // create a symlink to file File link = new File(del, LINK); FileUtil.symLink(tmpFile.toString(), link.toString()); // create a symlink to dir File linkDir = new File(del, "tmpDir"); FileUtil.symLink(tmp.toString(), linkDir.toString()); Assert.assertEquals(5, del.listFiles().length); // create files in partitioned directories createFile(partitioned, "part-r-00000", "foo"); createFile(partitioned, "part-r-00001", "bar");
5178 -1009652528apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc Even though 'y' is dangling symlink to file tmp/x, fullyDelete(y) should delete 'y' properly. Even though 'y' is dangling symlink to file tmp/x, fullyDelete(y) should delete 'y' properly. FILE_PATH_CHANGED setupDirs() private void setupDirs() throws IOException Assert.assertFalse(del.exists()); Assert.assertFalse(tmp.exists()); Assert.assertFalse(partitioned.exists()); del.mkdirs(); tmp.mkdirs(); partitioned.mkdirs(); new File(del, FILE).createNewFile(); File tmpFile = new File(tmp, FILE); tmpFile.createNewFile(); // create directories dir1.mkdirs(); dir2.mkdirs(); new File(dir1, FILE).createNewFile(); new File(dir2, FILE).createNewFile(); // create a symlink to file File link = new File(del, LINK); FileUtil.symLink(tmpFile.toString(), link.toString()); // create a symlink to dir File linkDir = new File(del, "tmpDir"); FileUtil.symLink(tmp.toString(), linkDir.toString()); Assert.assertEquals(5, del.listFiles().length); // create files in partitioned directories createFile(partitioned, "part-r-00000", "foo"); createFile(partitioned, "part-r-00001", "bar");
5177 -1009652587apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc move over the separator for next search move over the separator for next search FILE_PATH_CHANGED split(String, char, char) public static String[] split(String str, char escapeChar, char separator) if (str == null) { return null; } ArrayList strList = new ArrayList(); StringBuilder split = new StringBuilder(); int index = 0; while ((index = findNext(str, separator, escapeChar, index, split)) >= 0) { // move over the separator for next search ++index; strList.add(split.toString()); // reset the buffer split.setLength(0); } strList.add(split.toString()); // remove trailing empty split(s) // last split int last = strList.size(); while (--last >= 0 && "".equals(strList.get(last))) { strList.remove(last); } return strList.toArray(new String[strList.size()]);
5176 -1009652534apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc Benchmark for StringUtils split Benchmark for StringUtils split FILE_PATH_CHANGED main(String[]) public static void main(String[] args) final String TO_SPLIT = "foo,bar,baz,blah,blah"; for (boolean useOurs : new boolean[] { false, true }) { for (int outer = 0; outer < 10; outer++) { long st = System.nanoTime(); int components = 0; for (int inner = 0; inner < 1000000; inner++) { String[] res; if (useOurs) { res = StringUtils.split(TO_SPLIT, ','); } else { res = TO_SPLIT.split(","); } // be sure to use res, otherwise might be optimized out components += res.length; } long et = System.nanoTime(); if (outer > 3) { System.out.println((useOurs ? "StringUtils impl" : "Java impl") + " #" + outer + ":" + (et - st) / 1000000 + "ms"); } } }
5175 -1009652530apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc FIXME? Inflater docs say: 'it is also necessary to provide an extra \"dummy\" byte as input. This is required by the ZLIB native library in order to support certain optimizations.' However, this does not appear to be true, and in any case, it's not entirely clear where the byte should go or what its value should be. Perhaps it suffices to have some deflated bytes in the first buffer load? (But how else would one do it?) FIXME? Inflater docs say: 'it is also necessary to provide an extra \"dummy\" byte as input. This is required by the ZLIB native library in order to support certain optimizations.' However, this does not appear to be true, and in any case, it's not entirely clear where the byte should go or what its value should be. Perhaps it suffices to have some deflated bytes in the first buffer load? (But how else would one do it?) FILE_PATH_CHANGED needsInput() public synchronized boolean needsInput() if (state == GzipStateLabel.DEFLATE_STREAM) { // most common case return inflater.needsInput(); } // see userBufLen comment at top of decompress(); currently no need to // verify userBufLen <= 0 return (state != GzipStateLabel.FINISHED);
5174 -1009652525apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc Check if zlib consumed all input buffer set keepUncompressedBuf properly zlib consumed all input buffer Check if zlib consumed all input buffer set keepUncompressedBuf properly zlib consumed all input buffer FILE_PATH_CHANGED compress(byte[], int, int) public synchronized int compress(byte[] b, int off, int len) throws IOException if (b == null) { throw new NullPointerException(); } if (off < 0 || len < 0 || off > b.length - len) { throw new ArrayIndexOutOfBoundsException(); } int n = 0; // Check if there is compressed data n = compressedDirectBuf.remaining(); if (n > 0) { n = Math.min(n, len); ((ByteBuffer) compressedDirectBuf).get(b, off, n); return n; } // Re-initialize the zlib's output direct buffer compressedDirectBuf.rewind(); compressedDirectBuf.limit(directBufferSize); // Compress data n = deflateBytesDirect(); compressedDirectBuf.limit(n); // Check if zlib consumed all input buffer // set keepUncompressedBuf properly if (uncompressedDirectBufLen <= 0) { // zlib consumed all input buffer keepUncompressedBuf = false; uncompressedDirectBuf.clear(); uncompressedDirectBufOff = 0; uncompressedDirectBufLen = 0; } else { // zlib did not consume all input buffer keepUncompressedBuf = true; } // Get atmost 'len' bytes n = Math.min(n, len); ((ByteBuffer) compressedDirectBuf).get(b, off, n); return n;
5172 -1009652560apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc the directory better be empty the directory better be empty FILE_PATH_CHANGED rmdir(String) public int rmdir(String path) throws IOException if (isDirectory(path)) { // the directory better be empty String[] dirEntries = readdir(path); if ((dirEntries.length <= 2) && (localFS.delete(new Path(path), true))) return 0; } return -1;
5171 -1009652499apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc TO DO: - more efficient to not split the path, but simply compare TO DO: - more efficient to not split the path, but simply compare FILE_PATH_CHANGED resolve(String, boolean) // TO DO: - more efficient to not split the path, but simply compare String[] path = breakIntoPathComponents(p); if (path.length <= 1) { // special case for when path is "/" ResolveResult res = new ResolveResult(ResultKind.isInternalDir, root.InodeDirFs, root.fullPath, SlashPath); return res; } INodeDir curInode = root; int i; // ignore first slash for (i = 1; i < path.length - (resolveLastComponent ? 0 : 1); i++) { INode nextInode = curInode.resolveInternal(path[i]); if (nextInode == null) { StringBuilder failedAt = new StringBuilder(path[0]); for (int j = 1; j <= i; ++j) { failedAt.append('/').append(path[j]); } throw (new FileNotFoundException(failedAt.toString())); } if (nextInode instanceof INodeLink) { final INodeLink link = (INodeLink) nextInode; final Path remainingPath; if (i >= path.length - 1) { remainingPath = SlashPath; } else { StringBuilder remainingPathStr = new StringBuilder("/" + path[i + 1]); for (int j = i + 2; j < path.length; ++j) { remainingPathStr.append('/').append(path[j]); } remainingPath = new Path(remainingPathStr.toString()); } final ResolveResult res = new ResolveResult(ResultKind.isExternalDir, link.targetFileSystem, nextInode.fullPath, remainingPath); return res; } else if (nextInode instanceof INodeDir) { curInode = (INodeDir) nextInode; } } // We have resolved to an internal dir in mount table. Path remainingPath; if (resolveLastComponent) { remainingPath = SlashPath; } else { // note we have taken care of when path is "/" above // for internal dirs rem-path does not start with / since the lookup // that follows will do a children.get(remaningPath) and will have to // strip-out the initial / StringBuilder remainingPathStr = new StringBuilder("/" + path[i]); for (int j = i + 1; j < path.length; ++j) { remainingPathStr.append('/').append(path[j]); } remainingPath = new Path(remainingPathStr.toString()); } final ResolveResult res = new ResolveResult(ResultKind.isInternalDir, curInode.InodeDirFs, curInode.fullPath, remainingPath); return res; ResolveResult resolve(final String p, final boolean resolveLastComponent) throws FileNotFoundException
5170 -1009652595apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc Ideally we should wait after transferTo returns 0. But because of a bug in JRE on Linux (http://bugs.sun.com/view_bug.do?bug_id=5103988), which throws an exception instead of returning 0, we wait for the channel to be writable before writing to it. If you ever see IOException with message \"Resource temporarily unavailable\" thrown here, please let us know. Once we move to JAVA SE 7, wait should be moved to correct place. Ideally we should wait after transferTo returns 0. But because of a bug in JRE on Linux (http://bugs.sun.com/view_bug.do?bug_id=5103988), which throws an exception instead of returning 0, we wait for the channel to be writable before writing to it. If you ever see IOException with message \"Resource temporarily unavailable\" thrown here, please let us know. Once we move to JAVA SE 7, wait should be moved to correct place. FILE_PATH_CHANGED transferToFully(FileChannel, long, int) public void transferToFully(FileChannel fileCh, long position, int count) throws IOException while (count > 0) { /* * Ideally we should wait after transferTo returns 0. But because of * a bug in JRE on Linux (http://bugs.sun.com/view_bug.do?bug_id=5103988), * which throws an exception instead of returning 0, we wait for the * channel to be writable before writing to it. If you ever see * IOException with message "Resource temporarily unavailable" * thrown here, please let us know. * * Once we move to JAVA SE 7, wait should be moved to correct place. */ waitForWritable(); int nTransfered = (int) fileCh.transferTo(position, count, getChannel()); if (nTransfered == 0) { // check if end of file is reached. if (position >= fileCh.size()) { throw new EOFException("EOF Reached. file size is " + fileCh.size() + " and " + count + " more bytes left to be " + "transfered."); } // otherwise assume the socket is full. // waitForWritable(); // see comment above. } else if (nTransfered < 0) { throw new IOException("Unexpected return of " + nTransfered + " from transferTo()"); } else { position += nTransfered; count -= nTransfered; } }
5169 -1009652476apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc TODO: this should be abstract in a base class TODO: this should be abstract in a base class FILE_PATH_CHANGED getFS() protected FileSystem getFS() throws IOException if (fs == null) { fs = FileSystem.get(getConf()); } return fs;
5168 -1009652477apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc instance.run catches IOE, so something is REALLY wrong if here instance.run catches IOE, so something is REALLY wrong if here FILE_PATH_CHANGED getFS() protected FileSystem getFS() throws IOException if (fs == null) { fs = FileSystem.get(getConf()); } return fs;
5167 -1009652502apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc TODO: will eventually auto-wrap the text, but this matches the expected output for the hdfs tests... TODO: will eventually auto-wrap the text, but this matches the expected output for the hdfs tests... FILE_PATH_CHANGED getFS() protected FileSystem getFS() throws IOException if (fs == null) { fs = FileSystem.get(getConf()); } return fs;
5166 -1009652478apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc TODO: DFSAdmin subclasses FsShell so need to protect the command registration. This class should morph into a base class for commands, and then this method can be abstract TODO: DFSAdmin subclasses FsShell so need to protect the command registration. This class should morph into a base class for commands, and then this method can be abstract FILE_PATH_CHANGED getFS() protected FileSystem getFS() throws IOException if (fs == null) { fs = FileSystem.get(getConf()); } return fs;
5165 -1009652536apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc Authorization is done. Just call super. Authorization is done. Just call super. FILE_PATH_CHANGED doGet(HttpServletRequest, HttpServletResponse) protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException // Do the authorization if (HttpServer.hasAdministratorAccess(getServletContext(), request, response)) { // Authorization is done. Just call super. super.doGet(request, response); }
5163 -1009652606apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc If they try to bind to a different host's address, give a better error message. If they try to bind to a different host's address, give a better error message. FILE_PATH_CHANGED getProtocolClass(String, Configuration) static Class getProtocolClass(String protocolName, Configuration conf) throws ClassNotFoundException Class protocol = PROTOCOL_CACHE.get(protocolName); if (protocol == null) { protocol = conf.getClassByName(protocolName); PROTOCOL_CACHE.put(protocolName, protocol); } return protocol;
5161 -1009652557apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc TODO: sample the generated key/value records, and put the numbers below TODO: sample the generated key/value records, and put the numbers below FILE_PATH_CHANGED setUp() public void setUp() throws IOException skip = !(Algorithm.LZO.isSupported()); if (skip) { System.out.println("Skipped"); } // TODO: sample the generated key/value records, and put the numbers below init(Compression.Algorithm.LZO.getName(), "memcmp", "TFileTestCodecsLzo", 2605, 2558); if (!skip) super.setUp();
5159 -1009652487apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc TODO: handle this TODO: handle this FILE_PATH_CHANGED processPath(PathData) protected void processPath(PathData item) throws IOException if (item.stat.isDirectory()) { // TODO: handle this throw new PathIsDirectoryException(item.toString()); } if (item.stat.getLen() != 0) { throw new PathIOException(item.toString(), "Not a zero-length file"); } touchz(item);
5158 -1009652611apache/hadoopEli Collinsa196766ea07775f18ded69bd9e8d239f8cfd3ccc Read characters into a portion of an array, reading from the underlying stream at most once if necessary. Read characters into a portion of an array, reading from the underlying stream at most once if necessary. FILE_PATH_CHANGED read1(byte[], int, int) private int read1(byte[] b, int off, int len) throws IOException int avail = count - pos; if (avail <= 0) { if (len >= maxChunkSize) { // read a chunk to user buffer directly; avoid one copy int nread = readChecksumChunk(b, off, len); return nread; } else { // read a chunk into the local buffer fill(); if (count <= 0) { return -1; } else { avail = count; } } } // copy content of the local buffer to the user buffer int cnt = (avail < len) ? avail : len; System.arraycopy(buf, pos, b, off, cnt); pos += cnt; return cnt;
5157 -1009652476apache/hadoopEli Collins44a35b5d9accc4ecf7b1bbf762e593540bafe6a3 None TODO: this should be abstract in a base class SATD_ADDED getFS() protected FileSystem getFS() throws IOException if (fs == null) { fs = FileSystem.get(getConf()); } return fs;
5156 -1009652477apache/hadoopEli Collins44a35b5d9accc4ecf7b1bbf762e593540bafe6a3 None instance.run catches IOE, so something is REALLY wrong if here SATD_ADDED getFS() protected FileSystem getFS() throws IOException if (fs == null) { fs = FileSystem.get(getConf()); } return fs;
5155 -1009652478apache/hadoopEli Collins44a35b5d9accc4ecf7b1bbf762e593540bafe6a3 None TODO: DFSAdmin subclasses FsShell so need to protect the command registration. This class should morph into a base class for commands, and then this method can be abstract SATD_ADDED getFS() protected FileSystem getFS() throws IOException if (fs == null) { fs = FileSystem.get(getConf()); } return fs;
5153 -1009652507apache/hadoopEli Collins44a35b5d9accc4ecf7b1bbf762e593540bafe6a3 TODO: This isn't the best place, but this class is being abused with subclasses which of course override this method. There really needs to be a better base class for all commands initialize FsShell SATD_REMOVED getFS() protected FileSystem getFS() throws IOException if (fs == null) { fs = FileSystem.get(getConf()); } return fs;
5152 -1009652479apache/hadoopEli Collinsbabd19de331c875a1dffee908617c07c3e1eb31b None Ignored the mbean itself was not found, which should never happen because we just accessed it (perhaps something unregistered in-between) but if this happens just don't output the attribute. SATD_ADDED writeAttribute(JsonGenerator, ObjectName, MBeanAttributeInfo) private void writeAttribute(JsonGenerator jg, ObjectName oname, MBeanAttributeInfo attr) throws IOException if (!attr.isReadable()) { return; } String attName = attr.getName(); if ("modelerType".equals(attName)) { return; } if (attName.indexOf("=") >= 0 || attName.indexOf(":") >= 0 || attName.indexOf(" ") >= 0) { return; } Object value = null; try { value = mBeanServer.getAttribute(oname, attName); } catch (AttributeNotFoundException e) { // Ignored the attribute was not found, which should never happen because the bean // just told us that it has this attribute, but if this happens just don't output // the attribute. return; } catch (MBeanException e) { // The code inside the attribute getter threw an exception so log it, and // skip outputting the attribute LOG.error("getting attribute " + attName + " of " + oname + " threw an exception", e); return; } catch (RuntimeException e) { // For some reason even with an MBeanException available to them Runtime exceptions // can still find their way through, so treat them the same as MBeanException LOG.error("getting attribute " + attName + " of " + oname + " threw an exception", e); return; } catch (ReflectionException e) { // This happens when the code inside the JMX bean (setter?? from the java docs) // threw an exception, so log it and skip outputting the attribute LOG.error("getting attribute " + attName + " of " + oname + " threw an exception", e); return; } catch (InstanceNotFoundException e) { // Ignored the mbean itself was not found, which should never happen because we // just accessed it (perhaps something unregistered in-between) but if this // happens just don't output the attribute. return; } writeAttribute(jg, attName, value);
5151 -1009652480apache/hadoopJitendra Nath Pandeye05a6d1dce1b83bc2abe4eb2efc1afd51c8facd2 None Doing a second call with faults disabled should return fine -- ie the internal state of the client or server should not be broken by the failed call SATD_ADDED doErrorTest(Class, Class, Class, Class) private void doErrorTest(Class clientParamClass, Class serverParamClass, Class serverResponseClass, Class clientResponseClass) throws Exception // start server Server server = new TestServer(1, false, serverParamClass, serverResponseClass); InetSocketAddress addr = NetUtils.getConnectAddress(server); server.start(); // start client WRITABLE_FAULTS_ENABLED = true; Client client = new Client(clientResponseClass, conf); try { LongWritable param = clientParamClass.newInstance(); try { client.call(param, addr, null, null, 0, conf); fail("Expected an exception to have been thrown"); } catch (Throwable t) { assertExceptionContains(t, "Injected fault"); } // Doing a second call with faults disabled should return fine -- // ie the internal state of the client or server should not be broken // by the failed call WRITABLE_FAULTS_ENABLED = false; client.call(param, addr, null, null, 0, conf); } finally { server.stop(); }
5150 -1009652481apache/hadoopTodd Lipcon77b4fd6572d6f928ea5bd86c8b00caeba7bb3b99 None NOTE: this logic should be better, mimics previous implementation SATD_ADDED registerCommands(CommandFactory) public static void registerCommands(CommandFactory factory) factory.addClass(Merge.class, "-getmerge"); factory.addClass(Cp.class, "-cp"); factory.addClass(CopyFromLocal.class, "-copyFromLocal"); factory.addClass(CopyToLocal.class, "-copyToLocal"); factory.addClass(Get.class, "-get"); factory.addClass(Put.class, "-put");
5149 -1009652482apache/hadoopTodd Lipcon77b4fd6572d6f928ea5bd86c8b00caeba7bb3b99 None too bad we can't tell exactly why it failed... SATD_ADDED registerCommands(CommandFactory) public static void registerCommands(CommandFactory factory) factory.addClass(Merge.class, "-getmerge"); factory.addClass(Cp.class, "-cp"); factory.addClass(CopyFromLocal.class, "-copyFromLocal"); factory.addClass(CopyToLocal.class, "-copyToLocal"); factory.addClass(Get.class, "-get"); factory.addClass(Put.class, "-put");
5148 -1009652483apache/hadoopTodd Lipcon77b4fd6572d6f928ea5bd86c8b00caeba7bb3b99 None TODO: this really should be a -nl option SATD_ADDED registerCommands(CommandFactory) public static void registerCommands(CommandFactory factory) factory.addClass(Merge.class, "-getmerge"); factory.addClass(Cp.class, "-cp"); factory.addClass(CopyFromLocal.class, "-copyFromLocal"); factory.addClass(CopyToLocal.class, "-copyToLocal"); factory.addClass(Get.class, "-get"); factory.addClass(Put.class, "-put");
5147 -1009652484apache/hadoopTodd Lipcon77b4fd6572d6f928ea5bd86c8b00caeba7bb3b99 None TODO: commands should implement a -f to enable this SATD_ADDED setOverwrite(boolean) protected void setOverwrite(boolean flag) overwrite = flag;
5143 -1009652485apache/hadoopTsz-wo Sze7f77fad79af0010cd22ca773d9af27110429d3a2 None TODO: should probably allow path arguments for the filesystems SATD_ADDED processOptions(LinkedList) protected void processOptions(LinkedList args) throws IOException CommandFormat cf = new CommandFormat(null, 0, 0); cf.parse(args);
5142 -1009652486apache/hadoopTsz-wo Sze7f77fad79af0010cd22ca773d9af27110429d3a2 None TODO: if the user wants the trash to be used but there is any problem (ie. creating the trash dir, moving the item to be deleted, etc), then the path will just be deleted because moveToTrash returns false and it falls thru to fs.delete. this doesn't seem right SATD_ADDED registerCommands(CommandFactory) public static void registerCommands(CommandFactory factory) factory.addClass(Rm.class, "-rm"); factory.addClass(Rmr.class, "-rmr"); factory.addClass(Expunge.class, "-expunge");
5141 -1009652487apache/hadoopTsz-wo Szecd2079f0e4aa292492b5d6c0d0af5bfa41a39043 TODO: handle this TODO: handle this SATD_MOVED_FILE processPath(PathData) protected void processPath(PathData item) throws IOException if (item.stat.isDirectory()) { // TODO: handle this throw new PathIsDirectoryException(item.toString()); } if (item.stat.getLen() != 0) { throw new PathIOException(item.toString(), "Not a zero-length file"); } touchz(item);
5140 -1009652494apache/hadoopSuresh Srinivas8a2b40d0726215e48b53ab22382dd49379c36249 placeholder for javadoc to prevent broken links, until HADOOP-6920 race protection SATD_REMOVED getThreadUsage(MetricsRecordBuilder) private void getThreadUsage(MetricsRecordBuilder rb) int threadsNew = 0; int threadsRunnable = 0; int threadsBlocked = 0; int threadsWaiting = 0; int threadsTimedWaiting = 0; int threadsTerminated = 0; long[] threadIds = threadMXBean.getAllThreadIds(); for (ThreadInfo threadInfo : threadMXBean.getThreadInfo(threadIds, 0)) { // race protection if (threadInfo == null) continue; switch(threadInfo.getThreadState()) { case NEW: threadsNew++; break; case RUNNABLE: threadsRunnable++; break; case BLOCKED: threadsBlocked++; break; case WAITING: threadsWaiting++; break; case TIMED_WAITING: threadsTimedWaiting++; break; case TERMINATED: threadsTerminated++; break; } } rb.addGauge(ThreadsNew, threadsNew).addGauge(ThreadsRunnable, threadsRunnable).addGauge(ThreadsBlocked, threadsBlocked).addGauge(ThreadsWaiting, threadsWaiting).addGauge(ThreadsTimedWaiting, threadsTimedWaiting).addGauge(ThreadsTerminated, threadsTerminated);
5139 -1009652488apache/hadoopJitendra Nath Pandeya5290c9eca69027cff2448d05fee6983cbb54cd7 None TODO: this is a quick workaround to accelerate the integration of redesigned commands. this will be removed this once all commands are converted. this change will avoid having to change the hdfs tests every time a command is converted to use path-based exceptions SATD_ADDED getFS() protected FileSystem getFS() throws IOException if (fs == null) fs = FileSystem.get(getConf()); return fs;
5138 -1009652489apache/hadoopJitendra Nath Pandeya5290c9eca69027cff2448d05fee6983cbb54cd7 None TODO: it's backwards compat, but why is this throwing an exception? it's not like the shell test cmd SATD_ADDED getFS() protected FileSystem getFS() throws IOException if (fs == null) fs = FileSystem.get(getConf()); return fs;
5131 -1009652491apache/hadoopTsz-wo Sze3337cdb3121d926301a3cca17abef029abdb2ff3 None TODO: this is a pretty inconsistent way to output the path...!! but, it's backwards compatible SATD_ADDED registerCommands(CommandFactory) public static void registerCommands(CommandFactory factory) factory.addClass(Cat.class, "-cat"); factory.addClass(Text.class, "-text");
5129 -1009652492apache/hadoopTsz-wo Sze4de502c7c050373efe8620b320ab4413bd54cfa2 None TODO: this really should be a -nl option SATD_ADDED processOptions(LinkedList) protected void processOptions(LinkedList args) throws IOException CommandFormat cf = new CommandFormat(null, 2, 3); cf.parse(args); // TODO: this really should be a -nl option if ((args.size() > 2) && Boolean.parseBoolean(args.removeLast())) { delimiter = "\n"; } else { delimiter = null; } Path path = new Path(args.removeLast()); dst = new PathData(path.getFileSystem(getConf()), path);
5128 -1009652493apache/hadoopSuresh Srinivas38ac23159dd0eea5a58928fbcff501cbd9ffdd5b None TODO: printing the path twice is silly for backwards compatibility SATD_ADDED registerCommands(CommandFactory) public static void registerCommands(CommandFactory factory) factory.addClass(Chmod.class, "-chmod"); factory.addClass(Chown.class, "-chown"); factory.addClass(Chgrp.class, "-chgrp");
5125 -1009652495apache/hadoopEli Collins827401a9b1d85fe1f443d21ebe9d917ba23153de None hack to avoid most of the \"innocuous\" races. SATD_ADDED start() void start() if (startMBeans) startMBeans();
5124 -1009652496apache/hadoopTsz-wo Szef1c74df922058e88791ed6971bbb96b53f6770f1 None Test resolvePath(p) TODO In the tests below replace fcView.getDefaultFileSystem().resolvePath() fcView.resolvePath() SATD_ADDED testResolvePathInternalPaths() public void testResolvePathInternalPaths() throws IOException Assert.assertEquals(new Path("/"), fcView.getDefaultFileSystem().resolvePath(new Path("/"))); Assert.assertEquals(new Path("/internalDir"), fcView.getDefaultFileSystem().resolvePath(new Path("/internalDir")));
5123 -1009652497apache/hadoopTsz-wo Szef1c74df922058e88791ed6971bbb96b53f6770f1 None this is the stupid semantics of FileSystem SATD_ADDED mkdirs(Path, FsPermission) public boolean mkdirs(Path dir, FsPermission permission) throws AccessControlException, FileAlreadyExistsException if (theInternalDir.isRoot & dir == null) { throw new FileAlreadyExistsException("/ already exits"); } // Note dir starts with / if (theInternalDir.children.containsKey(dir.toString().substring(1))) { // this is the stupid semantics of FileSystem return true; } throw readOnlyMountTable("mkdirs", dir);
5122 -1009652498apache/hadoopTsz-wo Szef1c74df922058e88791ed6971bbb96b53f6770f1 None The implementors of RawLocalFileSystem were trying to be very smart. They implement FileStatus#getOwener lazily -- the object returned is really a RawLocalFileSystem that expect the FileStatus#getPath to be unchanged so that it can get owner when needed. Hence we need to interpose a new ViewFileSystemFileStatus that works around. SATD_ADDED readOnlyMountTable(String, String) static AccessControlException readOnlyMountTable(final String operation, final String p) return new AccessControlException("InternalDir of ViewFileSystem is readonly; operation=" + operation + "Path=" + p);
5121 -1009652499apache/hadoopTsz-wo Szef1c74df922058e88791ed6971bbb96b53f6770f1 None TO DO: - more efficient to not split the path, but simply compare SATD_ADDED resolve(String, boolean) // TO DO: - more efficient to not split the path, but simply compare String[] path = breakIntoPathComponents(p); if (path.length <= 1) { // special case for when path is "/" ResolveResult res = new ResolveResult(ResultKind.isInternalDir, root.InodeDirFs, root.fullPath, SlashPath); return res; } INodeDir curInode = root; int i; // ignore first slash for (i = 1; i < path.length - (resolveLastComponent ? 0 : 1); i++) { INode nextInode = curInode.resolveInternal(path[i]); if (nextInode == null) { StringBuilder failedAt = new StringBuilder(path[0]); for (int j = 1; j <= i; ++j) { failedAt.append('/').append(path[j]); } throw (new FileNotFoundException(failedAt.toString())); } if (nextInode instanceof INodeLink) { final INodeLink link = (INodeLink) nextInode; final Path remainingPath; if (i >= path.length - 1) { remainingPath = SlashPath; } else { StringBuilder remainingPathStr = new StringBuilder("/" + path[i + 1]); for (int j = i + 2; j < path.length; ++j) { remainingPathStr.append('/').append(path[j]); } remainingPath = new Path(remainingPathStr.toString()); } final ResolveResult res = new ResolveResult(ResultKind.isExternalDir, link.targetFileSystem, nextInode.fullPath, remainingPath); return res; } else if (nextInode instanceof INodeDir) { curInode = (INodeDir) nextInode; } } // We have resolved to an internal dir in mount table. Path remainingPath; if (resolveLastComponent) { remainingPath = SlashPath; } else { // note we have taken care of when path is "/" above // for internal dirs rem-path does not start with / since the lookup // that follows will do a children.get(remaningPath) and will have to // strip-out the initial / StringBuilder remainingPathStr = new StringBuilder("/" + path[i]); for (int j = i + 1; j < path.length; ++j) { remainingPathStr.append('/').append(path[j]); } remainingPath = new Path(remainingPathStr.toString()); } final ResolveResult res = new ResolveResult(ResultKind.isInternalDir, curInode.InodeDirFs, curInode.fullPath, remainingPath); return res; ResolveResult resolve(final String p, final boolean resolveLastComponent) throws FileNotFoundException
5120 -1009652500apache/hadoopTsz-wo Szef1c74df922058e88791ed6971bbb96b53f6770f1 None The implementors of RawLocalFileSystem were trying to be very smart. They implement FileStatus#getOwener lazily -- the object returned is really a RawLocalFileSystem that expect the FileStatus#getPath to be unchanged so that it can get owner when needed. Hence we need to interpose a new ViewFsFileStatus that works around. SATD_ADDED readOnlyMountTable(String, String) static AccessControlException readOnlyMountTable(final String operation, final String p) return new AccessControlException("InternalDir of ViewFileSystem is readonly; operation=" + operation + "Path=" + p);
5119 -1009652501apache/hadoopJitendra Nath Pandey369a20391555f9c0ca9bd5384435be12770942aa None this is very ugly, but needed to avoid breaking hdfs tests... if a path has no authority, then the FileStatus from globStatus will add the \"-fs\" authority into the path, so we need to sub it back out to satisfy the tests SATD_ADDED expandAsGlob(String, Configuration) public static PathData[] expandAsGlob(String pattern, Configuration conf) throws IOException Path globPath = new Path(pattern); FileSystem fs = globPath.getFileSystem(conf); FileStatus[] stats = fs.globStatus(globPath); PathData[] items = null; if (stats == null) { // not a glob & file not found, so add the path with a null stat items = new PathData[] { new PathData(fs, pattern, null) }; } else if (// this is very ugly, but needed to avoid breaking hdfs tests... // if a path has no authority, then the FileStatus from globStatus // will add the "-fs" authority into the path, so we need to sub // it back out to satisfy the tests stats.length == 1 && stats[0].getPath().equals(fs.makeQualified(globPath))) { // if the fq path is identical to the pattern passed, use the pattern // to initialize the string value items = new PathData[] { new PathData(fs, pattern, stats[0]) }; } else { // convert stats to PathData items = new PathData[stats.length]; int i = 0; for (FileStatus stat : stats) { items[i++] = new PathData(fs, stat); } } return items;
5117 -1009652513apache/hadoopJitendra Nath Pandey369a20391555f9c0ca9bd5384435be12770942aa TODO: this should be more posix-like: ex. \"No such file or directory\" TODO: this should be more posix-like: ex. \"No such file or directory\" CLASS_OR_METHOD_CHANGED processNonexistentPath(PathData) protected void processNonexistentPath(PathData item) throws IOException // TODO: this should be more posix-like: ex. "No such file or directory" throw new FileNotFoundException(getFnfText(item.path));
5115 -1009652502apache/hadoopTodd Lipcon99ebad8e757e90f6e036fc213d99f82dec7b80d7 None TODO: will eventually auto-wrap the text, but this matches the expected output for the hdfs tests... SATD_ADDED getFS() protected FileSystem getFS() throws IOException if (fs == null) fs = FileSystem.get(getConf()); return fs;
5114 -1009652503apache/hadoopTodd Lipcon99ebad8e757e90f6e036fc213d99f82dec7b80d7 None TODO: remove when the error is commonized... SATD_ADDED getFnfText(Path) protected String getFnfText(Path path) return "Can not find listing for " + path;
5113 -1009652504apache/hadoopTodd Lipcon99ebad8e757e90f6e036fc213d99f82dec7b80d7 None TODO: -1 should be reserved for syntax error, 1 should be failure SATD_ADDED run(String...) public int run(String... argv) LinkedList args = new LinkedList(Arrays.asList(argv)); try { processOptions(args); processArguments(expandArguments(args)); } catch (IOException e) { displayError(e); } // TODO: -1 should be reserved for syntax error, 1 should be failure return (numErrors == 0) ? exitCode : -1;
5112 -1009652505apache/hadoopTodd Lipcon99ebad8e757e90f6e036fc213d99f82dec7b80d7 None TODO: remove when the error is commonized... SATD_ADDED registerCommands(CommandFactory) public static void registerCommands(CommandFactory factory) factory.addClass(Ls.class, "-ls"); factory.addClass(Lsr.class, "-lsr");
5111 -1009652506apache/hadoopTodd Lipcon99ebad8e757e90f6e036fc213d99f82dec7b80d7 None implicitly recurse once for cmdline directories SATD_ADDED registerCommands(CommandFactory) public static void registerCommands(CommandFactory factory) factory.addClass(Ls.class, "-ls"); factory.addClass(Lsr.class, "-lsr");
5110 -1009652507apache/hadoopTsz-wo Sze6e74a3592cbfa976f758dbf50654a1d00d23c270 TODO: This isn't the best place, but this class is being abused with subclasses which of course override this method. There really needs to be a better base class for all commands TODO: This isn't the best place, but this class is being abused with subclasses which of course override this method. There really needs to be a better base class for all commands CLASS_OR_METHOD_CHANGED getFS() protected FileSystem getFS() throws IOException if (fs == null) fs = FileSystem.get(getConf()); return fs;
5109 -1009652596apache/hadoopTsz-wo Sze6e74a3592cbfa976f758dbf50654a1d00d23c270 TODO: handle this TODO: handle this CLASS_OR_METHOD_CHANGED getFS() protected FileSystem getFS() throws IOException if (fs == null) fs = FileSystem.get(getConf()); return fs;
5108 -1009652597apache/hadoopTsz-wo Sze6e74a3592cbfa976f758dbf50654a1d00d23c270 cat behavior in Linux [~/1207]$ ls ?.txt x.txt z.txt [~/1207]$ cat x.txt y.txt z.txt xxx cat: y.txt: No such file or directory zzz cat behavior in Linux [~/1207]$ ls ?.txt x.txt z.txt [~/1207]$ cat x.txt y.txt z.txt xxx cat: y.txt: No such file or directory zzz CLASS_OR_METHOD_CHANGED getFS() protected FileSystem getFS() throws IOException if (fs == null) fs = FileSystem.get(getConf()); return fs;
5107 -1009652598apache/hadoopTsz-wo Sze6e74a3592cbfa976f758dbf50654a1d00d23c270 Keep the structure similar to ChecksumFileSystem.copyToLocal(). Ideal these two should just invoke FileUtil.copy() and not repeat recursion here. Of course, copy() should support two more options : copyCrc and useTmpFile (may be useTmpFile need not be an option). Keep the structure similar to ChecksumFileSystem.copyToLocal(). Ideal these two should just invoke FileUtil.copy() and not repeat recursion here. Of course, copy() should support two more options : copyCrc and useTmpFile (may be useTmpFile need not be an option). CLASS_OR_METHOD_CHANGED getFS() protected FileSystem getFS() throws IOException if (fs == null) fs = FileSystem.get(getConf()); return fs;
5106 -1009652507apache/hadoopJitendra Nath Pandeyd358eb75b79b17f85ae9fd831a0bd065b87bf924 None TODO: This isn't the best place, but this class is being abused with subclasses which of course override this method. There really needs to be a better base class for all commands SATD_ADDED init() protected void init() throws IOException getConf().setQuietMode(true); if (this.fs == null) { this.fs = FileSystem.get(getConf()); } if (this.trash == null) { this.trash = new Trash(getConf()); }
5103 -1009652508apache/hadoopKonstantin Shvachkoa65753ddac34a114c51cb0010ee39a9af48b4f9e None TODO: will change with factory SATD_ADDED init() protected void init() throws IOException getConf().setQuietMode(true); if (this.fs == null) { this.fs = FileSystem.get(getConf()); } if (this.trash == null) { this.trash = new Trash(getConf()); }
5102 -1009652509apache/hadoopKonstantin Shvachkoa65753ddac34a114c51cb0010ee39a9af48b4f9e None TODO: next two lines are a temporary crutch until this entire block is overhauled SATD_ADDED init() protected void init() throws IOException getConf().setQuietMode(true); if (this.fs == null) { this.fs = FileSystem.get(getConf()); } if (this.trash == null) { this.trash = new Trash(getConf()); }
5101 -1009652510apache/hadoopKonstantin Shvachkoa65753ddac34a114c51cb0010ee39a9af48b4f9e None this is an unexpected condition, so dump the whole exception since it's probably a nasty internal error where the backtrace would be useful SATD_ADDED displayError(Exception) public void displayError(Exception e) // build up a list of exceptions that occurred exceptions.add(e); String errorMessage = e.getLocalizedMessage(); if (errorMessage == null) { // this is an unexpected condition, so dump the whole exception since // it's probably a nasty internal error where the backtrace would be // useful errorMessage = StringUtils.stringifyException(e); LOG.debug(errorMessage); } else { errorMessage = errorMessage.split("\n", 2)[0]; } displayError(errorMessage);
5100 -1009652511apache/hadoopKonstantin Shvachkoa65753ddac34a114c51cb0010ee39a9af48b4f9e None glob failed to match TODO: this should be more posix-like: ex. \"No such file or directory\" SATD_ADDED expandGlob(String) protected List expandGlob(String pattern) throws IOException Path path = new Path(pattern); FileSystem fs = path.getFileSystem(getConf()); FileStatus[] stats = fs.globStatus(path); if (stats != null && stats.length == 0) { // glob failed to match // TODO: this should be more posix-like: ex. "No such file or directory" throw new FileNotFoundException("Can not find listing for " + pattern); } List items = new LinkedList(); if (stats == null) { // not a glob & file not found, so null stat block items.add(new PathData(fs, path, null)); } else { // convert all the stats to PathData objs for (FileStatus stat : stats) { items.add(new PathData(fs, stat)); } } return items;
5099 -1009652512apache/hadoopKonstantin Shvachkoa65753ddac34a114c51cb0010ee39a9af48b4f9e None TODO: this really should be iterative SATD_ADDED processPaths(PathData, PathData...) protected void processPaths(PathData parent, PathData... items) throws IOException // TODO: this really should be iterative for (PathData item : items) { try { processPath(item); if (recursive && item.stat.isDirectory()) { recursePath(item); } } catch (IOException e) { displayError(e); } }
5098 -1009652513apache/hadoopKonstantin Shvachkoa65753ddac34a114c51cb0010ee39a9af48b4f9e None TODO: this should be more posix-like: ex. \"No such file or directory\" SATD_ADDED processNonexistentPathArgument(PathData) protected void processNonexistentPathArgument(PathData item) throws IOException // TODO: this should be more posix-like: ex. "No such file or directory" throw new FileNotFoundException("Can not find listing for " + item);
5094 -1009652514apache/hadoopTsz-wo Sze50b1f9fc73bedd7b5bd5d7c7ec1a43b17dd117ac None TODO: next two lines are a temporary crutch until this entire block is overhauled SATD_ADDED init() protected void init() throws IOException getConf().setQuietMode(true); if (this.fs == null) { this.fs = FileSystem.get(getConf()); } if (this.trash == null) { this.trash = new Trash(getConf()); }
5093 -1009652515apache/hadoopTsz-wo Sze50b1f9fc73bedd7b5bd5d7c7ec1a43b17dd117ac None this is an unexpected condition, so dump the whole exception since it's probably a nasty internal error where the backtrace would be useful SATD_ADDED displayError(Exception) public void displayError(Exception e) // build up a list of exceptions that occurred exceptions.add(e); String errorMessage = e.getLocalizedMessage(); if (errorMessage == null) { // this is an unexpected condition, so dump the whole exception since // it's probably a nasty internal error where the backtrace would be // useful errorMessage = StringUtils.stringifyException(e); LOG.debug(errorMessage); } else { errorMessage = errorMessage.split("\n", 2)[0]; } displayError(errorMessage);
5092 -1009652516apache/hadoopTsz-wo Sze50b1f9fc73bedd7b5bd5d7c7ec1a43b17dd117ac None TODO: this really should be iterative SATD_ADDED processPaths(PathData, PathData...) protected void processPaths(PathData parent, PathData... items) throws IOException // TODO: this really should be iterative for (PathData item : items) { try { processPath(item); if (recursive && item.stat.isDirectory()) { recursePath(item); } } catch (IOException e) { displayError(e); } }
5091 -1009652517apache/hadoopTsz-wo Sze0d55e1a14430ee18a84de6f985da86dc61d7ae80 None Read the int[] object as written by ObjectWritable, but \"going around\" ObjectWritable SATD_ADDED testOldFormat() public void testOldFormat() throws IOException // Make sure we still correctly write the old format if desired. // Write the data array with old ObjectWritable API // which will set allowCompactArrays false. ObjectWritable.writeObject(out, i, i.getClass(), null); // Get ready to read it back in.reset(out.getData(), out.getLength()); // Read the int[] object as written by ObjectWritable, but // "going around" ObjectWritable @SuppressWarnings("deprecation") String className = UTF8.readString(in); assertEquals("The int[] written by ObjectWritable as a non-compact array " + "was not labelled as an array of int", i.getClass().getName(), className); int length = in.readInt(); assertEquals("The int[] written by ObjectWritable as a non-compact array " + "was not expected length", i.length, length); int[] readValue = new int[length]; try { for (int i = 0; i < length; i++) { readValue[i] = (int) ((Integer) ObjectWritable.readObject(in, null)); } } catch (Exception e) { fail("The int[] written by ObjectWritable as a non-compact array " + "was corrupted. Failed to correctly read int[] of length " + length + ". Got exception:\n" + StringUtils.stringifyException(e)); } assertTrue("The int[] written by ObjectWritable as a non-compact array " + "was corrupted.", Arrays.equals(i, readValue));
5090 -1009652518apache/hadoopTsz-wo Sze0d55e1a14430ee18a84de6f985da86dc61d7ae80 None Read the APW object as written by ObjectWritable, but \"going around\" ObjectWritable SATD_ADDED testObjectLabeling() public void testObjectLabeling() throws IOException // Do a few tricky experiments to make sure things are being written // the way we expect // Write the data array with ObjectWritable // which will indirectly write it using APW.Internal ObjectWritable.writeObject(out, i, i.getClass(), null, true); // Write the corresponding APW directly with ObjectWritable ArrayPrimitiveWritable apw = new ArrayPrimitiveWritable(i); ObjectWritable.writeObject(out, apw, apw.getClass(), null, true); // Get ready to read it back in.reset(out.getData(), out.getLength()); // Read the int[] object as written by ObjectWritable, but // "going around" ObjectWritable String className = UTF8.readString(in); assertEquals("The int[] written by ObjectWritable was not labelled as " + "an ArrayPrimitiveWritable.Internal", ArrayPrimitiveWritable.Internal.class.getName(), className); ArrayPrimitiveWritable.Internal apwi = new ArrayPrimitiveWritable.Internal(); apwi.readFields(in); assertEquals("The ArrayPrimitiveWritable.Internal component type was corrupted", int.class, apw.getComponentType()); assertTrue("The int[] written by ObjectWritable as " + "ArrayPrimitiveWritable.Internal was corrupted", Arrays.equals(i, (int[]) (apwi.get()))); // Read the APW object as written by ObjectWritable, but // "going around" ObjectWritable String declaredClassName = UTF8.readString(in); assertEquals("The APW written by ObjectWritable was not labelled as " + "declaredClass ArrayPrimitiveWritable", ArrayPrimitiveWritable.class.getName(), declaredClassName); className = UTF8.readString(in); assertEquals("The APW written by ObjectWritable was not labelled as " + "class ArrayPrimitiveWritable", ArrayPrimitiveWritable.class.getName(), className); ArrayPrimitiveWritable apw2 = new ArrayPrimitiveWritable(); apw2.readFields(in); assertEquals("The ArrayPrimitiveWritable component type was corrupted", int.class, apw2.getComponentType()); assertTrue("The int[] written by ObjectWritable as " + "ArrayPrimitiveWritable was corrupted", Arrays.equals(i, (int[]) (apw2.get())));
5089 -1009652519apache/hadoopTsz-wo Sze0d55e1a14430ee18a84de6f985da86dc61d7ae80 None Read the int[] object as written by ObjectWritable, but \"going around\" ObjectWritable SATD_ADDED testObjectLabeling() public void testObjectLabeling() throws IOException // Do a few tricky experiments to make sure things are being written // the way we expect // Write the data array with ObjectWritable // which will indirectly write it using APW.Internal ObjectWritable.writeObject(out, i, i.getClass(), null, true); // Write the corresponding APW directly with ObjectWritable ArrayPrimitiveWritable apw = new ArrayPrimitiveWritable(i); ObjectWritable.writeObject(out, apw, apw.getClass(), null, true); // Get ready to read it back in.reset(out.getData(), out.getLength()); // Read the int[] object as written by ObjectWritable, but // "going around" ObjectWritable String className = UTF8.readString(in); assertEquals("The int[] written by ObjectWritable was not labelled as " + "an ArrayPrimitiveWritable.Internal", ArrayPrimitiveWritable.Internal.class.getName(), className); ArrayPrimitiveWritable.Internal apwi = new ArrayPrimitiveWritable.Internal(); apwi.readFields(in); assertEquals("The ArrayPrimitiveWritable.Internal component type was corrupted", int.class, apw.getComponentType()); assertTrue("The int[] written by ObjectWritable as " + "ArrayPrimitiveWritable.Internal was corrupted", Arrays.equals(i, (int[]) (apwi.get()))); // Read the APW object as written by ObjectWritable, but // "going around" ObjectWritable String declaredClassName = UTF8.readString(in); assertEquals("The APW written by ObjectWritable was not labelled as " + "declaredClass ArrayPrimitiveWritable", ArrayPrimitiveWritable.class.getName(), declaredClassName); className = UTF8.readString(in); assertEquals("The APW written by ObjectWritable was not labelled as " + "class ArrayPrimitiveWritable", ArrayPrimitiveWritable.class.getName(), className); ArrayPrimitiveWritable apw2 = new ArrayPrimitiveWritable(); apw2.readFields(in); assertEquals("The ArrayPrimitiveWritable component type was corrupted", int.class, apw2.getComponentType()); assertTrue("The int[] written by ObjectWritable as " + "ArrayPrimitiveWritable was corrupted", Arrays.equals(i, (int[]) (apw2.get())));
5088 -1009652520apache/hadoopTsz-wo Sze0d55e1a14430ee18a84de6f985da86dc61d7ae80 None For efficient implementation, there's no way around the following massive code duplication. SATD_ADDED writeBooleanArray(DataOutput) private void writeBooleanArray(DataOutput out) throws IOException boolean[] v = (boolean[]) value; for (int i = 0; i < length; i++) out.writeBoolean(v[i]);
5086 -1009652535apache/hadoopKonstantin Shvachkob1e3037296e5687ef04e86a35e1f00195faab244 perhaps a VIP is failing over perhaps a VIP is failing over CLASS_OR_METHOD_CHANGED waitForProtocolProxy(Class, long, InetSocketAddress, Configuration, int, long) public static ProtocolProxy waitForProtocolProxy(Class protocol, long clientVersion, InetSocketAddress addr, Configuration conf, int rpcTimeout, long timeout) throws IOException long startTime = System.currentTimeMillis(); IOException ioe; while (true) { try { return getProtocolProxy(protocol, clientVersion, addr, UserGroupInformation.getCurrentUser(), conf, NetUtils.getDefaultSocketFactory(conf), rpcTimeout); } catch (ConnectException se) { // namenode has not been started LOG.info("Server at " + addr + " not available yet, Zzzzz..."); ioe = se; } catch (SocketTimeoutException te) { // namenode is busy LOG.info("Problem connecting to server: " + addr); ioe = te; } catch (NoRouteToHostException nrthe) { // perhaps a VIP is failing over LOG.info("No route to host for server: " + addr); ioe = nrthe; } // check if timed out if (System.currentTimeMillis() - timeout >= startTime) { throw ioe; } // wait for retry try { Thread.sleep(1000); } catch (InterruptedException ie) { // IGNORE } }
5085 -1009652522apache/hadoopNigel Daley23da8fe75e474f2ece36bc23c7368f12bc9a56cc None better safe than sorry (should never happen) SATD_ADDED cacheGroupsAdd(List) public void cacheGroupsAdd(List groups) throws IOException for (String group : groups) { if (group.length() == 0) { // better safe than sorry (should never happen) } else if (group.charAt(0) == '@') { if (!NetgroupCache.isCached(group)) { NetgroupCache.add(group, getUsersForNetgroup(group)); } } else { // unix group, not caching } }
5084 -1009652523apache/hadoopNigel Daley23da8fe75e474f2ece36bc23c7368f12bc9a56cc None better safe than sorry (should never happen) SATD_ADDED cacheGroupsAdd(List) public void cacheGroupsAdd(List groups) throws IOException for (String group : groups) { if (group.length() == 0) { // better safe than sorry (should never happen) } else if (group.charAt(0) == '@') { if (!NetgroupCache.isCached(group)) { NetgroupCache.add(group, getUsersForNetgroup(group)); } } else { // unix group, not caching } }
5082 -1009652525apache/hadoopEli Collins8bd9dd0f336f9bd1c6a14a6689a5b0bf6face4f0 None Check if zlib consumed all input buffer set keepUncompressedBuf properly zlib consumed all input buffer SATD_ADDED compress(byte[], int, int) public synchronized int compress(byte[] b, int off, int len) throws IOException if (b == null) { throw new NullPointerException(); } if (off < 0 || len < 0 || off > b.length - len) { throw new ArrayIndexOutOfBoundsException(); } int n = 0; // Check if there is compressed data n = compressedDirectBuf.remaining(); if (n > 0) { n = Math.min(n, len); ((ByteBuffer) compressedDirectBuf).get(b, off, n); return n; } // Re-initialize the zlib's output direct buffer compressedDirectBuf.rewind(); compressedDirectBuf.limit(directBufferSize); // Compress data n = deflateBytesDirect(); compressedDirectBuf.limit(n); // Check if zlib consumed all input buffer // set keepUncompressedBuf properly if (uncompressedDirectBufLen <= 0) { // zlib consumed all input buffer keepUncompressedBuf = false; uncompressedDirectBuf.clear(); uncompressedDirectBufOff = 0; uncompressedDirectBufLen = 0; } else { // zlib did not consume all input buffer keepUncompressedBuf = true; } // Get atmost 'len' bytes n = Math.min(n, len); ((ByteBuffer) compressedDirectBuf).get(b, off, n); return n;
5081 -1009652526apache/hadoopSanjay Radiafa87ae80581f40dcdd71724a4df4f9e185106d72 None The following XDR recipe was done through a careful reading of gm_protocol.x in Ganglia 3.1 and carefully examining the output of the gmetric utility with strace. SATD_ADDED emitMetric(String, String, String) protected void emitMetric(String name, String type, String value) throws IOException if (name == null) { LOG.warn("Metric was emitted with no name."); return; } else if (value == null) { LOG.warn("Metric name " + name + " was emitted with a null value."); return; } else if (type == null) { LOG.warn("Metric name " + name + ", value " + value + " has no type."); return; } LOG.debug("Emitting metric " + name + ", type " + type + ", value " + value + " from hostname" + hostName); String units = getUnits(name); if (units == null) { LOG.warn("Metric name " + name + ", value " + value + " had 'null' units"); units = ""; } int slope = getSlope(name); int tmax = getTmax(name); int dmax = getDmax(name); offset = 0; String groupName = name.substring(0, name.lastIndexOf(".")); // The following XDR recipe was done through a careful reading of // gm_protocol.x in Ganglia 3.1 and carefully examining the output of // the gmetric utility with strace. // First we send out a metadata message // metric_id = metadata_msg xdr_int(128); // hostname xdr_string(hostName); // metric name xdr_string(name); // spoof = False xdr_int(0); // metric type xdr_string(type); // metric name xdr_string(name); // units xdr_string(units); // slope xdr_int(slope); // tmax, the maximum time between metrics xdr_int(tmax); // dmax, the maximum data value xdr_int(dmax); xdr_int(1); /*Num of the entries in extra_value field for Ganglia 3.1.x*/ xdr_string("GROUP"); /*Group attribute*/ xdr_string(groupName); /*Group value*/ for (SocketAddress socketAddress : metricsServers) { DatagramPacket packet = new DatagramPacket(buffer, offset, socketAddress); datagramSocket.send(packet); } // Now we send out a message with the actual value. // Technically, we only need to send out the metadata message once for // each metric, but I don't want to have to record which metrics we did and // did not send. offset = 0; // we are sending a string value xdr_int(133); // hostName xdr_string(hostName); // metric name xdr_string(name); // spoof = False xdr_int(0); // format field xdr_string("%s"); // metric value xdr_string(value); for (SocketAddress socketAddress : metricsServers) { DatagramPacket packet = new DatagramPacket(buffer, offset, socketAddress); datagramSocket.send(packet); }
5080 -1009652535apache/hadoopDevaraj Das714e5f7165b101f72a43e8c3fb27be5def93fe42 perhaps a VIP is failing over perhaps a VIP is failing over CLASS_OR_METHOD_CHANGED waitForProxy(Class, long, InetSocketAddress, Configuration, int, long) public static Object waitForProxy(Class protocol, long clientVersion, InetSocketAddress addr, Configuration conf, int rpcTimeout, long timeout) throws IOException long startTime = System.currentTimeMillis(); IOException ioe; while (true) { try { return getProxy(protocol, clientVersion, addr, UserGroupInformation.getCurrentUser(), conf, NetUtils.getDefaultSocketFactory(conf), rpcTimeout); } catch (ConnectException se) { // namenode has not been started LOG.info("Server at " + addr + " not available yet, Zzzzz..."); ioe = se; } catch (SocketTimeoutException te) { // namenode is busy LOG.info("Problem connecting to server: " + addr); ioe = te; } catch (NoRouteToHostException nrthe) { // perhaps a VIP is failing over LOG.info("No route to host for server: " + addr); ioe = nrthe; } // check if timed out if (System.currentTimeMillis() - timeout >= startTime) { throw ioe; } // wait for retry try { Thread.sleep(1000); } catch (InterruptedException ie) { // IGNORE } }
5079 -1009652527apache/hadoopDevaraj Das2c0598a9646bb8a43375d2dee83c0c60cbfc52df None Even though tmpDir is dangling symlink to tmp, fullyDelete(tmpDir) should delete tmpDir properly. SATD_ADDED setupDirs() private void setupDirs() throws IOException Assert.assertFalse(del.exists()); Assert.assertFalse(tmp.exists()); del.mkdirs(); tmp.mkdirs(); new File(del, FILE).createNewFile(); File tmpFile = new File(tmp, FILE); tmpFile.createNewFile(); // create directories dir1.mkdirs(); dir2.mkdirs(); new File(dir1, FILE).createNewFile(); new File(dir2, FILE).createNewFile(); // create a symlink to file File link = new File(del, LINK); FileUtil.symLink(tmpFile.toString(), link.toString()); // create a symlink to dir File linkDir = new File(del, "tmpDir"); FileUtil.symLink(tmp.toString(), linkDir.toString()); Assert.assertEquals(5, del.listFiles().length);
5078 -1009652528apache/hadoopDevaraj Das2c0598a9646bb8a43375d2dee83c0c60cbfc52df None Even though 'y' is dangling symlink to file tmp/x, fullyDelete(y) should delete 'y' properly. SATD_ADDED setupDirs() private void setupDirs() throws IOException Assert.assertFalse(del.exists()); Assert.assertFalse(tmp.exists()); del.mkdirs(); tmp.mkdirs(); new File(del, FILE).createNewFile(); File tmpFile = new File(tmp, FILE); tmpFile.createNewFile(); // create directories dir1.mkdirs(); dir2.mkdirs(); new File(dir1, FILE).createNewFile(); new File(dir2, FILE).createNewFile(); // create a symlink to file File link = new File(del, LINK); FileUtil.symLink(tmpFile.toString(), link.toString()); // create a symlink to dir File linkDir = new File(del, "tmpDir"); FileUtil.symLink(tmp.toString(), linkDir.toString()); Assert.assertEquals(5, del.listFiles().length);
5077 -1009652529apache/hadoopJakob Homan2a248dfc32e5061c1f80295f448ca525ade764c6 None NOTE: this is the one place we do NOT want to save the number of bytes sent (nRemaining here) into lastBytesSent: since we are resending what we've already sent before, offset is nonzero in general (only way it could be zero is if it already equals nRemaining), which would then screw up the offset calculation _next_ time around. IOW, getRemaining() is in terms of the original, zero-offset bufferload, so lastBytesSent must be as well. Cheesy ASCII art: <------------ m, lastBytesSent -----------> +===============================================+ buffer: |1111111111|22222222222222222|333333333333| | +===============================================+ #1: <-- off -->|<-------- nRemaining ---------> #2: <----------- off ----------->|<-- nRem. --> SATD_ADDED read() public int read() throws IOException checkStream(); return (read(oneByte, 0, oneByte.length) == -1) ? -1 : (oneByte[0] & 0xff);
5076 -1009652530apache/hadoopJakob Homan2a248dfc32e5061c1f80295f448ca525ade764c6 None FIXME? Inflater docs say: 'it is also necessary to provide an extra \"dummy\" byte as input. This is required by the ZLIB native library in order to support certain optimizations.' However, this does not appear to be true, and in any case, it's not entirely clear where the byte should go or what its value should be. Perhaps it suffices to have some deflated bytes in the first buffer load? (But how else would one do it?) SATD_ADDED needsInput() public synchronized boolean needsInput() if (state == GzipStateLabel.DEFLATE_STREAM) { // most common case return inflater.needsInput(); } // see userBufLen comment at top of decompress(); currently no need to // verify userBufLen <= 0 return (state != GzipStateLabel.FINISHED);
5069 -1009652533apache/hadoopBoris Shkolnik69693b6a8625dba4c783c61e9b49cfd722e1a74e None XXX Twenty internal version does not support --script option. SATD_ADDED getCommand(String, String) protected String[] getCommand(String command, String confDir) ArrayList cmdArgs = new ArrayList(); File binDir = getBinDir(); cmdArgs.add("ssh"); cmdArgs.add(hostName); cmdArgs.add(binDir.getAbsolutePath() + File.separator + SCRIPT_NAME); cmdArgs.add("--config"); cmdArgs.add(confDir); // XXX Twenty internal version does not support --script option. cmdArgs.add(command); cmdArgs.add(daemonName); return (String[]) cmdArgs.toArray(new String[cmdArgs.size()]);
5068 -1009652534apache/hadoopChristopher Douglasfe34ccdda84575045022d3041aab42481f636ddc None Benchmark for StringUtils split SATD_ADDED main(String[]) public static void main(String[] args) final String TO_SPLIT = "foo,bar,baz,blah,blah"; for (boolean useOurs : new boolean[] { false, true }) { for (int outer = 0; outer < 10; outer++) { long st = System.nanoTime(); int components = 0; for (int inner = 0; inner < 1000000; inner++) { String[] res; if (useOurs) { res = StringUtils.split(TO_SPLIT, ','); } else { res = TO_SPLIT.split(","); } // be sure to use res, otherwise might be optimized out components += res.length; } long et = System.nanoTime(); if (outer > 3) { System.out.println((useOurs ? "StringUtils impl" : "Java impl") + " #" + outer + ":" + (et - st) / 1000000 + "ms"); } } }
5067 -1009652535apache/hadoopHairong Kuang2bec54de58ca8c5feaa24aee6444e1d7d507b14f None perhaps a VIP is failing over SATD_ADDED waitForProxy(Class, long, InetSocketAddress, Configuration, long) public static Object waitForProxy(Class protocol, long clientVersion, InetSocketAddress addr, Configuration conf, long timeout) throws IOException long startTime = System.currentTimeMillis(); IOException ioe; while (true) { try { return getProxy(protocol, clientVersion, addr, conf); } catch (ConnectException se) { // namenode has not been started LOG.info("Server at " + addr + " not available yet, Zzzzz..."); ioe = se; } catch (SocketTimeoutException te) { // namenode is busy LOG.info("Problem connecting to server: " + addr); ioe = te; } catch (NoRouteToHostException nrthe) { // perhaps a VIP is failing over LOG.info("No route to host for server: " + addr); ioe = nrthe; } // check if timed out if (System.currentTimeMillis() - timeout >= startTime) { throw ioe; } // wait for retry try { Thread.sleep(1000); } catch (InterruptedException ie) { // IGNORE } }
5066 -1009652536apache/hadoopOwen O'Malley7785232da5a1db2d06f5d208ce3258c342a4d19c None Authorization is done. Just call super. SATD_ADDED doGet(HttpServletRequest, HttpServletResponse) protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException // Do the authorization if (HttpServer.hasAdministratorAccess(getServletContext(), request, response)) { // Authorization is done. Just call super. super.doGet(request, response); }
5065 -1009652537apache/hadoopDevaraj Das1793e7d9094cd984ae402177c5935239059d74e8 None Authorize the superuser which is doing doAs @param user ugi of the effective or proxy user which contains a real user @param remoteAddress the ip address of client @param conf configuration @throws AuthorizationException SATD_ADDED authorize(UserGroupInformation, String, Configuration) public static void authorize(UserGroupInformation user, String remoteAddress, Configuration conf) throws AuthorizationException if (user.getRealUser() == null) { return; } boolean groupAuthorized = false; UserGroupInformation superUser = user.getRealUser(); Collection allowedUserGroups = conf.getStringCollection(getProxySuperuserGroupConfKey(superUser.getShortUserName())); if (!allowedUserGroups.isEmpty()) { for (String group : user.getGroupNames()) { if (allowedUserGroups.contains(group)) { groupAuthorized = true; break; } } } if (!groupAuthorized) { throw new AuthorizationException("User: " + superUser.getUserName() + " is not allowed to impersonate " + user.getUserName()); } Collection ipList = conf.getStringCollection(getProxySuperuserIpConfKey(superUser.getShortUserName())); if (!ipList.isEmpty()) { for (String allowedHost : ipList) { InetAddress hostAddr; try { hostAddr = InetAddress.getByName(allowedHost); } catch (UnknownHostException e) { continue; } if (hostAddr.getHostAddress().equals(remoteAddress)) { // Authorization is successful return; } } } throw new AuthorizationException("Unauthorized connection for super-user: " + superUser.getUserName() + " from IP " + remoteAddress);
5064 -1009652538apache/hadoopHairong Kuang0c5734e4aca873f405fbf994e5fe7061e31731c8 None Flip to the newly parsed permissions SATD_ADDED refresh(Configuration, PolicyProvider) public static synchronized void refresh(Configuration conf, PolicyProvider provider) // Get the system property 'hadoop.policy.file' String policyFile = System.getProperty("hadoop.policy.file", HADOOP_POLICY_FILE); // Make a copy of the original config, and load the policy file Configuration policyConf = new Configuration(conf); policyConf.addResource(policyFile); final Map, AccessControlList> newAcls = new IdentityHashMap, AccessControlList>(); // Parse the config file Service[] services = provider.getServices(); if (services != null) { for (Service service : services) { AccessControlList acl = new AccessControlList(policyConf.get(service.getServiceKey(), AccessControlList.WILDCARD_ACL_VALUE)); newAcls.put(service.getProtocol(), acl); } } // Flip to the newly parsed permissions protocolToAcl = newAcls;
5061 -1009652539apache/hadoopThomas White4cecab7c10758cde1036b436d874b2b3a686a874 None Asking the CodecPool for a decompressor for GzipCodec should return null as well. SATD_ADDED testCodecPoolAndGzipDecompressor() public void testCodecPoolAndGzipDecompressor() // BuiltInZlibInflater should not be used as the GzipCodec decompressor. // Assert that this is the case. // Don't use native libs for this test. Configuration conf = new Configuration(); conf.setBoolean("hadoop.native.lib", false); assertFalse("ZlibFactory is using native libs against request", ZlibFactory.isNativeZlibLoaded(conf)); // This should give us a BuiltInZlibInflater. Decompressor zlibDecompressor = ZlibFactory.getZlibDecompressor(conf); assertNotNull("zlibDecompressor is null!", zlibDecompressor); assertTrue("ZlibFactory returned unexpected inflator", zlibDecompressor instanceof BuiltInZlibInflater); // Asking for a decompressor directly from GzipCodec should return null; // its createOutputStream() just wraps the existing stream in a // java.util.zip.GZIPOutputStream. CompressionCodecFactory ccf = new CompressionCodecFactory(conf); CompressionCodec codec = ccf.getCodec(new Path("foo.gz")); assertTrue("Codec for .gz file is not GzipCodec", codec instanceof GzipCodec); Decompressor codecDecompressor = codec.createDecompressor(); if (null != codecDecompressor) { fail("Got non-null codecDecompressor: " + codecDecompressor); } // Asking the CodecPool for a decompressor for GzipCodec // should return null as well. Decompressor poolDecompressor = CodecPool.getDecompressor(codec); if (null != poolDecompressor) { fail("Got non-null poolDecompressor: " + poolDecompressor); } // If we then ensure that the pool is populated... CodecPool.returnDecompressor(zlibDecompressor); // Asking the pool another time should still not bind this to GzipCodec. poolDecompressor = CodecPool.getDecompressor(codec); if (null != poolDecompressor) { fail("Second time, got non-null poolDecompressor: " + poolDecompressor); }
5060 -1009652605apache/hadoopSuresh Srinivasae93ba7501d95e9d26a29de25f4cc39e5225ca20 TODO: Get the user name from the GSS API for Kerberbos-based security Create the user subject server-side, don't trust the user groups provided by the client SATD_REMOVED getProtocolClass(String, Configuration) static Class getProtocolClass(String protocolName, Configuration conf) throws ClassNotFoundException Class protocol = PROTOCOL_CACHE.get(protocolName); if (protocol == null) { protocol = conf.getClassByName(protocolName); PROTOCOL_CACHE.put(protocolName, protocol); } return protocol;
5059 -1009652605apache/hadoopSuresh Srinivasae93ba7501d95e9d26a29de25f4cc39e5225ca20 TODO: Get the user name from the GSS API for Kerberbos-based security Create the user subject TODO: Get the user name from the GSS API for Kerberbos-based security SATD_CHANGED getProtocolClass(String, Configuration) static Class getProtocolClass(String protocolName, Configuration conf) throws ClassNotFoundException Class protocol = PROTOCOL_CACHE.get(protocolName); if (protocol == null) { protocol = conf.getClassByName(protocolName); PROTOCOL_CACHE.put(protocolName, protocol); } return protocol;
5058 -1009652544apache/hadoopThomas Whiteea5db0c5ab1b80a90aa3f2e2527b8a71e377c605 Some filesystem like HDFS ignore the \"x\" bit if the permission. Others like localFs does not. Override the method below if the file system being tested masks our certain bits for file masks. Some filesystem like HDFS ignore the \"x\" bit if the permission. Others like localFs does not. Override the method below if the file system being tested masks our certain bits for file masks. CLASS_OR_METHOD_CHANGED setUp() public void setUp() throws Exception fc.mkdir(getTestRootPath(fc), FileContext.DEFAULT_PERM, true);
5057 -1009652540apache/hadoopThomas White7a9ec5d90b55076c1e59249f13ac75030090b55a This may have been a misparse. java.net.URI specifies that a URI is of the form: URI ::= [SCHEME-PART:]SCHEME-SPECIFIC-PART The scheme-specific-part may be parsed in numerous ways, but if it starts with a '/' character, that makes it a \"hierarchical URI\", subject to the following parsing: SCHEME-SPECIFIC-PART ::= \"//\" AUTHORITY-PART AUTHORITY-PART ::= [USER-INFO-PART] HOSTNAME [ \":\" PORT ] In Hadoop, we require a host-based authority as well. java.net.URI parses left-to-right, so deprecated hostnames of the form 'foo:8020' will have 'foo' as their scheme and '8020' as their scheme-specific-part. We don't want this behavior. unqualified is \"hdfs://\" SATD_REMOVED get(Configuration) public static FileSystem get(Configuration conf) throws IOException return get(getDefaultUri(conf), conf);
5056 -1009652541apache/hadoopThomas White7a9ec5d90b55076c1e59249f13ac75030090b55a This may have been a misparse. java.net.URI specifies that a URI is of the form: URI ::= [SCHEME-PART:]SCHEME-SPECIFIC-PART The scheme-specific-part may be parsed in numerous ways, but if it starts with a '/' character, that makes it a \"hierarchical URI\", subject to the following parsing: SCHEME-SPECIFIC-PART ::= \"//\" AUTHORITY-PART AUTHORITY-PART ::= [USER-INFO-PART] HOSTNAME [ \":\" PORT ] In Hadoop, we require a host-based authority as well. java.net.URI parses left-to-right, so deprecated hostnames of the form 'foo:8020' will have 'foo' as their scheme and '8020' as their scheme-specific-part. We don't want this behavior. convert old-format name to new-format name \"local\" is now \"file:///\". SATD_REMOVED get(Configuration) public static FileSystem get(Configuration conf) throws IOException return get(getDefaultUri(conf), conf);
5055 -1009652541apache/hadoopThomas White74b46cf87e6b148e936b7cf3cbfbfd7aea1edf91 None This may have been a misparse. java.net.URI specifies that a URI is of the form: URI ::= [SCHEME-PART:]SCHEME-SPECIFIC-PART The scheme-specific-part may be parsed in numerous ways, but if it starts with a '/' character, that makes it a \"hierarchical URI\", subject to the following parsing: SCHEME-SPECIFIC-PART ::= \"//\" AUTHORITY-PART AUTHORITY-PART ::= [USER-INFO-PART] HOSTNAME [ \":\" PORT ] In Hadoop, we require a host-based authority as well. java.net.URI parses left-to-right, so deprecated hostnames of the form 'foo:8020' will have 'foo' as their scheme and '8020' as their scheme-specific-part. We don't want this behavior. SATD_ADDED get(Configuration) public static FileSystem get(Configuration conf) throws IOException return get(getDefaultUri(conf), conf);
5054 -1009652542apache/hadoopJakob Homan929e91a08c5387c692ed3257361190b83d72f2e9 None End of HADOOP-6386 workaround SATD_ADDED createBaseListener(Configuration) protected Connector createBaseListener(Configuration conf) throws IOException SelectChannelConnector ret = new SelectChannelConnector(); ret.setLowResourceMaxIdleTime(10000); ret.setAcceptQueueSize(128); ret.setResolveNames(false); ret.setUseDirectBuffers(false); return ret;
5053 -1009652543apache/hadoopJakob Homan929e91a08c5387c692ed3257361190b83d72f2e9 None Workaround for HADOOP-6386 SATD_ADDED createBaseListener(Configuration) protected Connector createBaseListener(Configuration conf) throws IOException SelectChannelConnector ret = new SelectChannelConnector(); ret.setLowResourceMaxIdleTime(10000); ret.setAcceptQueueSize(128); ret.setResolveNames(false); ret.setUseDirectBuffers(false); return ret;
5052 -1009652544apache/hadoopSuresh Srinivas3f371a0a644181b204111ee4e12c995fc7b5e5f5 None Some filesystem like HDFS ignore the \"x\" bit if the permission. Others like localFs does not. Override the method below if the file system being tested masks our certain bits for file masks. SATD_ADDED getTestRootRelativePath(String) protected Path getTestRootRelativePath(String pathString) return fc.makeQualified(new Path(TEST_ROOT_DIR, pathString));
5051 -1009652545apache/hadoopSuresh Srinivas3f371a0a644181b204111ee4e12c995fc7b5e5f5 None Default impl assumes that permissions do not matter calling the regular create is good enough. FSs that implement permissions should override this. SATD_ADDED createInternal(Path, EnumSet, FsPermission, int, short, long, Progressable, int, boolean) protected FSDataOutputStream createInternal(Path f, EnumSet flag, FsPermission absolutePermission, int bufferSize, short replication, long blockSize, Progressable progress, int bytesPerChecksum, boolean createParent) throws IOException checkPath(f); // Default impl assumes that permissions do not matter // calling the regular create is good enough. // FSs that implement permissions should override this. if (!createParent) { // parent must exist. // since this.create makes parent dirs automatically // we must throw exception if parent does not exist. final FileStatus stat = getFileStatus(f.getParent()); if (stat == null) { throw new FileNotFoundException("Missing parent:" + f); } if (!stat.isDir()) { throw new ParentNotDirectoryException("parent is not a dir:" + f); } // parent does exist - go ahead with create of file. } return fsImpl.primitiveCreate(f, absolutePermission, flag, bufferSize, replication, blockSize, progress, bytesPerChecksum);
5050 -1009652550apache/hadoopSuresh Srinivas3f371a0a644181b204111ee4e12c995fc7b5e5f5 passed to the filesystem below When we move to new AbstractFileSystem then it is not really needed except for default FS for this FileContext. Fully qualified SATD_REMOVED accept(Path) public boolean accept(final Path file) return true;
5049 -1009652546apache/hadoopTsz-wo Sze3e9ba3584153a41b0ceaeecd72407bd1069b161a None Set new default probability if specified through a system.property If neither is specified set default probability to DEFAULT_PROB SATD_ADDED injectCriteria(String) public static boolean injectCriteria(String klassName) boolean trigger = false; if (generator.nextFloat() < getProbability(klassName)) { trigger = true; } return trigger;
5047 -1009652548apache/hadoopSuresh Srinivasfa48d9ea17e754e7bf2d5d0f71d98bfa94d177e0 None Default impl is to assume that permissions do not matter and hence calling the regular mkdirs is good enough. FSs that implement permissions should override this. SATD_ADDED get(Configuration) public static FileSystem get(Configuration conf) throws IOException return get(getDefaultUri(conf), conf);
5046 -1009652549apache/hadoopSuresh Srinivasfa48d9ea17e754e7bf2d5d0f71d98bfa94d177e0 None Default impl assumes that permissions do not matter and nor does the bytesPerChecksum hence calling the regular create is good enough. FSs that implement permissions should override this. SATD_ADDED get(Configuration) public static FileSystem get(Configuration conf) throws IOException return get(getDefaultUri(conf), conf);
5045 -1009652548apache/hadoopThomas White0294c49df60150bd9b363af5cfbc312222c12c69 None Default impl is to assume that permissions do not matter and hence calling the regular mkdirs is good enough. FSs that implement permissions should override this. SATD_ADDED get(Configuration) public static FileSystem get(Configuration conf) throws IOException return get(getDefaultUri(conf), conf);
5044 -1009652549apache/hadoopThomas White0294c49df60150bd9b363af5cfbc312222c12c69 None Default impl assumes that permissions do not matter and nor does the bytesPerChecksum hence calling the regular create is good enough. FSs that implement permissions should override this. SATD_ADDED get(Configuration) public static FileSystem get(Configuration conf) throws IOException return get(getDefaultUri(conf), conf);
5042 -1009652551apache/hadoopTsz-wo Sze86724941c5144a06113090d40c50c289e7ebd290 None The following if clause handles the following case: Assume the following scenario in BZip2 compressed stream where . represent compressed data. .....[48 bit Block].....[48 bit Block].....[48 bit Block]... ........................[47 bits][1 bit].....[48 bit Block]... ................................^[Assume a Byte alignment here] ........................................^^[current position of stream] .....................^^[We go back 10 Bytes in stream and find a Block marker] ........................................^^[We align at wrong position!] ...........................................................^^[While this pos is correct] SATD_ADDED createOutputStream(OutputStream) public CompressionOutputStream createOutputStream(OutputStream out) throws IOException return new BZip2CompressionOutputStream(out);
5038 -1009652552apache/hadoopMichael Stack412efb84f25b0f6caada9c698c3cb3456b81e006 None The following XDR recipe was done through a careful reading of gm_protocol.x in Ganglia 3.1 and carefully examining the output of the gmetric utility with strace. SATD_ADDED emitMetric(String, String, String) protected void emitMetric(String name, String type, String value) throws IOException if (name == null) { LOG.warn("Metric was emitted with no name."); return; } else if (value == null) { LOG.warn("Metric name " + name + " was emitted with a null value."); return; } else if (type == null) { LOG.warn("Metric name " + name + ", value " + value + " has no type."); return; } LOG.debug("Emitting metric " + name + ", type " + type + ", value " + value + " from hostname" + hostName); String units = getUnits(name); if (units == null) { LOG.warn("Metric name " + name + ", value " + value + " had 'null' units"); units = ""; } int slope = getSlope(name); int tmax = getTmax(name); int dmax = getDmax(name); offset = 0; String groupName = name.substring(0, name.lastIndexOf(".")); // The following XDR recipe was done through a careful reading of // gm_protocol.x in Ganglia 3.1 and carefully examining the output of // the gmetric utility with strace. // First we send out a metadata message // metric_id = metadata_msg xdr_int(128); // hostname xdr_string(hostName); // metric name xdr_string(name); // spoof = False xdr_int(0); // metric type xdr_string(type); // metric name xdr_string(name); // units xdr_string(units); // slope xdr_int(slope); // tmax, the maximum time between metrics xdr_int(tmax); // dmax, the maximum data value xdr_int(dmax); xdr_int(1); /*Num of the entries in extra_value field for Ganglia 3.1.x*/ xdr_string("GROUP"); /*Group attribute*/ xdr_string(groupName); /*Group value*/ for (SocketAddress socketAddress : metricsServers) { DatagramPacket packet = new DatagramPacket(buffer, offset, socketAddress); datagramSocket.send(packet); } // Now we send out a message with the actual value. // Technically, we only need to send out the metadata message once for // each metric, but I don't want to have to record which metrics we did and // did not send. offset = 0; // we are sending a string value xdr_int(133); // hostName xdr_string(hostName); // metric name xdr_string(name); // spoof = False xdr_int(0); // format field xdr_string("%s"); // metric value xdr_string(value); for (SocketAddress socketAddress : metricsServers) { DatagramPacket packet = new DatagramPacket(buffer, offset, socketAddress); datagramSocket.send(packet); }
5029 -1009652553apache/hadoopChristopher Douglas4824a54419f1ccb24aa0af07edfbcf1539084089 None Verify that skip trash option really skips the trash for rmr SATD_ADDED writeFile(FileSystem, Path) protected static Path writeFile(FileSystem fs, Path f) throws IOException DataOutputStream out = fs.create(f); out.writeBytes("dhruba: " + f); out.close(); assertTrue(fs.exists(f)); return f;
5028 -1009652554apache/hadoopChristopher Douglas4824a54419f1ccb24aa0af07edfbcf1539084089 None Verify skip trash option really works SATD_ADDED writeFile(FileSystem, Path) protected static Path writeFile(FileSystem fs, Path f) throws IOException DataOutputStream out = fs.create(f); out.writeBytes("dhruba: " + f); out.close(); assertTrue(fs.exists(f)); return f;
5027 -1009652555apache/hadoopThomas White8246aa28ff72e3ae81eb6ce59852abd5828fadc6 None TODO: remember the longest key in a TFile, and use it to replace MAX_KEY_SIZE. SATD_ADDED getChunkBufferSize(Configuration) static int getChunkBufferSize(Configuration conf) int ret = conf.getInt(CHUNK_BUF_SIZE_ATTR, 1024 * 1024); return (ret > 0) ? ret : 1024 * 1024;
5026 -1009652556apache/hadoopThomas White8246aa28ff72e3ae81eb6ce59852abd5828fadc6 None flag to ensure value is only examined once. SATD_ADDED getChunkBufferSize(Configuration) static int getChunkBufferSize(Configuration conf) int ret = conf.getInt(CHUNK_BUF_SIZE_ATTR, 1024 * 1024); return (ret > 0) ? ret : 1024 * 1024;
5025 -1009652557apache/hadoopThomas White8246aa28ff72e3ae81eb6ce59852abd5828fadc6 None TODO: sample the generated key/value records, and put the numbers below SATD_ADDED setUp() public void setUp() throws IOException skip = !(Algorithm.LZO.isSupported()); if (skip) { System.out.println("Skipped"); } // TODO: sample the generated key/value records, and put the numbers below init(Compression.Algorithm.LZO.getName(), "memcmp", "TFileTestCodecsLzo", 2605, 2558); if (!skip) super.setUp();
5024 -1009652558apache/hadoopThomas White8246aa28ff72e3ae81eb6ce59852abd5828fadc6 None move the cursor to the beginning of the tail, containing: offset to the meta block index, version and magic SATD_ADDED register(long, long, long) public void register(long raw, long offsetStart, long offsetEnd) mp
5023 -1009652559apache/hadoopOwen O'Malleyfb6308f280ad30a61757f988e21b424e64fce03f None Verify fix of bug identified in HADOOP-6004 SATD_ADDED testDeserialization() public void testDeserialization() throws IOException // Create a test BlockLocation String[] names = { "one", "two" }; String[] hosts = { "three", "four" }; String[] topologyPaths = { "five", "six" }; long offset = 25l; long length = 55l; BlockLocation bl = new BlockLocation(names, hosts, topologyPaths, offset, length); DataOutputBuffer dob = new DataOutputBuffer(); // Serialize it try { bl.write(dob); } catch (IOException e) { fail("Unable to serialize data: " + e.getMessage()); } byte[] bytes = dob.getData(); DataInput da = new DataInputStream(new ByteArrayInputStream(bytes)); // Try to re-create the BlockLocation the same way as is done during // deserialization BlockLocation bl2 = new BlockLocation(); try { bl2.readFields(da); } catch (IOException e) { fail("Unable to deserialize BlockLocation: " + e.getMessage()); } // Check that we got back what we started with verifyDeserialization(bl2.getHosts(), hosts); verifyDeserialization(bl2.getNames(), names); verifyDeserialization(bl2.getTopologyPaths(), topologyPaths); assertEquals(bl2.getOffset(), offset); assertEquals(bl2.getLength(), length);
5014 -1009652560apache/hadoopOwen O'Malleyac670cc47f06b0eb880dcb5522a5aba187b5c67d None the directory better be empty SATD_ADDED rmdir(String) public int rmdir(String path) throws IOException if (isDirectory(path)) { // the directory better be empty String[] dirEntries = readdir(path); if ((dirEntries.length <= 2) && (localFS.delete(new Path(path), true))) return 0; } return -1;
5013 -1009652561apache/hadoopOwen O'Malleyac670cc47f06b0eb880dcb5522a5aba187b5c67d None verify that after expunging the Trash, it really goes away SATD_ADDED writeFile(FileSystem, Path) protected static Path writeFile(FileSystem fs, Path f) throws IOException DataOutputStream out = fs.create(f); out.writeBytes("dhruba: " + f); out.close(); assertTrue(fs.exists(f)); return f;
5012 -1009652562apache/hadoopOwen O'Malleyac670cc47f06b0eb880dcb5522a5aba187b5c67d None Check that every permission has its sticky bit represented correctly SATD_ADDED testStickyBitToString() public void testStickyBitToString() // Check that every permission has its sticky bit represented correctly for (boolean sb : new boolean[] { false, true }) { for (FsAction u : FsAction.values()) { for (FsAction g : FsAction.values()) { for (FsAction o : FsAction.values()) { FsPermission f = new FsPermission(u, g, o, sb); if (f.getStickyBit() && f.getOtherAction().implies(EXECUTE)) assertEquals('t', f.toString().charAt(8)); else if (f.getStickyBit() && !f.getOtherAction().implies(EXECUTE)) assertEquals('T', f.toString().charAt(8)); else if (!f.getStickyBit() && f.getOtherAction().implies(EXECUTE)) assertEquals('x', f.toString().charAt(8)); else assertEquals('-', f.toString().charAt(8)); } } } }
5011 -1009652563apache/hadoopOwen O'Malley95a0db602b2e0606af11d666d9d10d64766f9ecf None Refresh the service level authorization policy once again, this time it should fail! SATD_ADDED testRefresh() public void testRefresh() throws Exception MiniDFSCluster dfs = null; try { final int slaves = 4; // Turn on service-level authorization Configuration conf = new Configuration(); conf.setClass(PolicyProvider.POLICY_PROVIDER_CONFIG, HDFSPolicyProvider.class, PolicyProvider.class); conf.setBoolean(ServiceAuthorizationManager.SERVICE_AUTHORIZATION_CONFIG, true); // Start the mini dfs cluster dfs = new MiniDFSCluster(conf, slaves, true, null); // Refresh the service level authorization policy refreshPolicy(conf); // Simulate an 'edit' of hadoop-policy.xml String confDir = System.getProperty("test.build.extraconf", "build/test/extraconf"); File policyFile = new File(confDir, ConfiguredPolicy.HADOOP_POLICY_FILE); String policyFileCopy = ConfiguredPolicy.HADOOP_POLICY_FILE + ".orig"; // first save original FileUtil.copy(// first save original policyFile, // first save original FileSystem.getLocal(conf), new Path(confDir, policyFileCopy), false, conf); rewriteHadoopPolicyFile(new File(confDir, ConfiguredPolicy.HADOOP_POLICY_FILE)); // Refresh the service level authorization policy refreshPolicy(conf); // Refresh the service level authorization policy once again, // this time it should fail! try { // Note: hadoop-policy.xml for tests has // security.refresh.policy.protocol.acl = ${user.name} conf.set(UnixUserGroupInformation.UGI_PROPERTY_NAME, UNKNOWN_USER); refreshPolicy(conf); fail("Refresh of NameNode's policy file cannot be successful!"); } catch (RemoteException re) { System.out.println("Good, refresh worked... refresh failed with: " + StringUtils.stringifyException(re.unwrapRemoteException())); } finally { // Reset to original hadoop-policy.xml FileUtil.fullyDelete(new File(confDir, ConfiguredPolicy.HADOOP_POLICY_FILE)); FileUtil.replaceFile(new File(confDir, policyFileCopy), new File(confDir, ConfiguredPolicy.HADOOP_POLICY_FILE)); } } finally { if (dfs != null) { dfs.shutdown(); } }
5010 -1009652564apache/hadoopOwen O'Malley95a0db602b2e0606af11d666d9d10d64766f9ecf None Refresh the service level authorization policy SATD_ADDED testRefresh() public void testRefresh() throws Exception MiniDFSCluster dfs = null; try { final int slaves = 4; // Turn on service-level authorization Configuration conf = new Configuration(); conf.setClass(PolicyProvider.POLICY_PROVIDER_CONFIG, HDFSPolicyProvider.class, PolicyProvider.class); conf.setBoolean(ServiceAuthorizationManager.SERVICE_AUTHORIZATION_CONFIG, true); // Start the mini dfs cluster dfs = new MiniDFSCluster(conf, slaves, true, null); // Refresh the service level authorization policy refreshPolicy(conf); // Simulate an 'edit' of hadoop-policy.xml String confDir = System.getProperty("test.build.extraconf", "build/test/extraconf"); File policyFile = new File(confDir, ConfiguredPolicy.HADOOP_POLICY_FILE); String policyFileCopy = ConfiguredPolicy.HADOOP_POLICY_FILE + ".orig"; // first save original FileUtil.copy(// first save original policyFile, // first save original FileSystem.getLocal(conf), new Path(confDir, policyFileCopy), false, conf); rewriteHadoopPolicyFile(new File(confDir, ConfiguredPolicy.HADOOP_POLICY_FILE)); // Refresh the service level authorization policy refreshPolicy(conf); // Refresh the service level authorization policy once again, // this time it should fail! try { // Note: hadoop-policy.xml for tests has // security.refresh.policy.protocol.acl = ${user.name} conf.set(UnixUserGroupInformation.UGI_PROPERTY_NAME, UNKNOWN_USER); refreshPolicy(conf); fail("Refresh of NameNode's policy file cannot be successful!"); } catch (RemoteException re) { System.out.println("Good, refresh worked... refresh failed with: " + StringUtils.stringifyException(re.unwrapRemoteException())); } finally { // Reset to original hadoop-policy.xml FileUtil.fullyDelete(new File(confDir, ConfiguredPolicy.HADOOP_POLICY_FILE)); FileUtil.replaceFile(new File(confDir, policyFileCopy), new File(confDir, ConfiguredPolicy.HADOOP_POLICY_FILE)); } } finally { if (dfs != null) { dfs.shutdown(); } }
5009 -1009652565apache/hadoopOwen O'Malley95a0db602b2e0606af11d666d9d10d64766f9ecf None rare case where splits are exact, logs.length can be 4 SATD_ADDED reset() void reset() final int oldsize = size; do { size = gen.nextInt(MAX_SIZE); } while (oldsize == size); final long oldseed = seed; do { seed = gen.nextLong() & Long.MAX_VALUE; } while (oldseed == seed);
5008 -1009652566apache/hadoopOwen O'Malleycab0a4bf543f58600ae8499f5b219a452f89c827 None TODO enable upload command only when selection is exactly one folder SATD_ADDED uploadDirectoryToDFS(IStructuredSelection) private void uploadDirectoryToDFS(IStructuredSelection selection) throws InvocationTargetException, InterruptedException // Ask the user which local directory to upload DirectoryDialog dialog = new DirectoryDialog(Display.getCurrent().getActiveShell(), SWT.OPEN | SWT.MULTI); dialog.setText("Select the local file or directory to upload"); String dirName = dialog.open(); final File dir = new File(dirName); List files = new ArrayList(); files.add(dir); // TODO enable upload command only when selection is exactly one folder final List folders = filterSelection(DFSFolder.class, selection); if (folders.size() >= 1) uploadToDFS(folders.get(0), files);
5007 -1009652567apache/hadoopOwen O'Malleycab0a4bf543f58600ae8499f5b219a452f89c827 None TODO enable upload command only when selection is exactly one folder SATD_ADDED uploadFilesToDFS(IStructuredSelection) private void uploadFilesToDFS(IStructuredSelection selection) throws InvocationTargetException, InterruptedException // Ask the user which files to upload FileDialog dialog = new FileDialog(Display.getCurrent().getActiveShell(), SWT.OPEN | SWT.MULTI); dialog.setText("Select the local files to upload"); dialog.open(); List files = new ArrayList(); for (String fname : dialog.getFileNames()) files.add(new File(dialog.getFilterPath() + File.separator + fname)); // TODO enable upload command only when selection is exactly one folder List folders = filterSelection(DFSFolder.class, selection); if (folders.size() >= 1) uploadToDFS(folders.get(0), files);
5006 -1009652568apache/hadoopOwen O'Malleycab0a4bf543f58600ae8499f5b219a452f89c827 None todo: only if all classes are of proper types SATD_ADDED isValid(ILaunchConfiguration) public boolean isValid(ILaunchConfiguration launchConfig) // todo: only if all classes are of proper types return true;
5005 -1009652569apache/hadoopOwen O'Malleycab0a4bf543f58600ae8499f5b219a452f89c827 None TODO close DFS connections? SATD_ADDED dispose() public void dispose() // TODO close DFS connections?
5004 -1009652570apache/hadoopOwen O'Malleycab0a4bf543f58600ae8499f5b219a452f89c827 None FIXME Error dialog SATD_ADDED findLaunchConfiguration(IType, ILaunchConfigurationType) protected ILaunchConfiguration findLaunchConfiguration(IType type, ILaunchConfigurationType configType) // Find an existing or create a launch configuration (Standard way) ILaunchConfiguration iConf = super.findLaunchConfiguration(type, configType); ILaunchConfigurationWorkingCopy iConfWC; try { /* * Tune the default launch configuration: setup run-time classpath * manually */ iConfWC = iConf.getWorkingCopy(); iConfWC.setAttribute(IJavaLaunchConfigurationConstants.ATTR_DEFAULT_CLASSPATH, false); List classPath = new ArrayList(); IResource resource = type.getResource(); IJavaProject project = (IJavaProject) resource.getProject().getNature(JavaCore.NATURE_ID); IRuntimeClasspathEntry cpEntry = JavaRuntime.newDefaultProjectClasspathEntry(project); classPath.add(0, cpEntry.getMemento()); iConfWC.setAttribute(IJavaLaunchConfigurationConstants.ATTR_CLASSPATH, classPath); } catch (CoreException e) { e.printStackTrace(); // FIXME Error dialog return null; } /* * Update the selected configuration with a specific Hadoop location * target */ IResource resource = type.getResource(); if (!(resource instanceof IFile)) return null; RunOnHadoopWizard wizard = new RunOnHadoopWizard((IFile) resource, iConfWC); WizardDialog dialog = new WizardDialog(Display.getDefault().getActiveShell(), wizard); dialog.create(); dialog.setBlockOnOpen(true); if (dialog.open() != WizardDialog.OK) return null; try { iConfWC.doSave(); } catch (CoreException e) { e.printStackTrace(); // FIXME Error dialog return null; } return iConfWC;
5003 -1009652571apache/hadoopOwen O'Malleycab0a4bf543f58600ae8499f5b219a452f89c827 None XXX don't know what the file is? SATD_ADDED upload(IProgressMonitor, File) public void upload(IProgressMonitor monitor, final File file) throws IOException if (file.isDirectory()) { Path filePath = new Path(this.path, file.getName()); getDFS().mkdirs(filePath); DFSFolder newFolder = new DFSFolder(this, filePath); monitor.worked(1); for (File child : file.listFiles()) { if (monitor.isCanceled()) return; newFolder.upload(monitor, child); } } else if (file.isFile()) { Path filePath = new Path(this.path, file.getName()); DFSFile newFile = new DFSFile(this, filePath, file, monitor); } else { // XXX don't know what the file is? }
5002 -1009652572apache/hadoopOwen O'Malleycab0a4bf543f58600ae8499f5b219a452f89c827 None TODO SATD_ADDED performFinish() public HadoopServer performFinish() try { if (this.original == null) { // New location Display.getDefault().syncExec(new Runnable() { public void run() { ServerRegistry.getInstance().addServer(HadoopLocationWizard.this.location); } }); return this.location; } else { // Update location final String originalName = this.original.getLocationName(); this.original.load(this.location); Display.getDefault().syncExec(new Runnable() { public void run() { ServerRegistry.getInstance().updateServer(originalName, HadoopLocationWizard.this.location); } }); return this.original; } } catch (Exception e) { e.printStackTrace(); setMessage("Invalid server location values", IMessageProvider.ERROR); return null; }
5001 -1009652573apache/hadoopOwen O'Malleyabe7be913432053f6d419ea4ca4f9cd2be938bc7 None verify that after expunging the Trash, it really goes away SATD_ADDED writeFile(FileSystem, Path) protected static Path writeFile(FileSystem fs, Path f) throws IOException DataOutputStream out = fs.create(f); out.writeBytes("dhruba: " + f); out.close(); assertTrue(fs.exists(f)); return f;
5000 -1009652574apache/hadoopOwen O'Malleyabe7be913432053f6d419ea4ca4f9cd2be938bc7 None Check that every permission has its sticky bit represented correctly SATD_ADDED testStickyBitToString() public void testStickyBitToString() // Check that every permission has its sticky bit represented correctly for (boolean sb : new boolean[] { false, true }) { for (FsAction u : FsAction.values()) { for (FsAction g : FsAction.values()) { for (FsAction o : FsAction.values()) { FsPermission f = new FsPermission(u, g, o, sb); if (f.getStickyBit() && f.getOtherAction().implies(EXECUTE)) assertEquals('t', f.toString().charAt(8)); else if (f.getStickyBit() && !f.getOtherAction().implies(EXECUTE)) assertEquals('T', f.toString().charAt(8)); else if (!f.getStickyBit() && f.getOtherAction().implies(EXECUTE)) assertEquals('x', f.toString().charAt(8)); else assertEquals('-', f.toString().charAt(8)); } } } }
4999 -1009652575apache/hadoopOwen O'Malleyabe7be913432053f6d419ea4ca4f9cd2be938bc7 None the directory better be empty SATD_ADDED rmdir(String) public int rmdir(String path) throws IOException if (isDirectory(path)) { // the directory better be empty String[] dirEntries = readdir(path); if ((dirEntries.length <= 2) && (localFS.delete(new Path(path), true))) return 0; } return -1;
4996 1242678131GerritCodeReview/gerritDave Borowitz2443905d177e6b9b1e3ead9d923b34bfcdc3cadb TODO(dborowitz): Re-enable when ConsistencyChecker supports notedb. TODO(dborowitz): Re-enable when ConsistencyChecker supports notedb. Note that we *do* want to enable these tests with GERRIT_CHECK_NOTEDB, as we need to be able to convert old, corrupt changes. However, those tests don't necessarily need to pass. SATD_REMOVED setUp() public void setUp() throws Exception // TODO(dborowitz): Re-enable when ConsistencyChecker supports notedb. // Note that we *do* want to enable these tests with GERRIT_CHECK_NOTEDB, as // we need to be able to convert old, corrupt changes. However, those tests // don't necessarily need to pass. assume().that(notesMigration.enabled()).isFalse(); // Ignore client clone of project; repurpose as server-side TestRepository. testRepo = new TestRepository<>((InMemoryRepository) repoManager.openRepository(project)); tip = testRepo.getRevWalk().parseCommit(testRepo.getRepository().exactRef("HEAD").getObjectId()); adminId = admin.getId(); checker = checkerProvider.get();
4995 1242678172GerritCodeReview/gerritDave Borowitzcf2fa7983f4046a97f01d58ae6dcd4f8bc095386 None TODO(dborowitz): Use a ThreadLocal or use Joda. SATD_ADDED changeRefName(Change.Id) public static String changeRefName(Change.Id id) StringBuilder r = new StringBuilder(); r.append(RefNames.REFS_CHANGES); int n = id.get(); int m = n % 100; if (m < 10) { r.append('0'); } r.append(m); r.append('/'); r.append(n); r.append(RefNames.META_SUFFIX); return r.toString();
4993 1242678171GerritCodeReview/gerritUrs Wolferf367b5d3aee7dc84e9be799646f4076d9db9a0d5 None We're not generally supposed to do work in provider constructors, but this is a bit of a special case because we really need to have the ID available by the time the dbInjector is created. This even applies during RebuildNoteDb, which otherwise would have been a reasonable place to do the ID generation. Fortunately, it's not much work, and it happens once. SATD_ADDED get() public String get() return id;
4990 1242678168GerritCodeReview/gerritEdwin Kempinf156a4d76a1521d9aeb8d029a5703e44cbef7b55 None TODO: Handle range comment SATD_ADDED newDraft(CodeMirror) private void newDraft(CodeMirror cm) int cmLine = cm.getLineNumber(cm.extras().activeLine()); LineSidePair pair = host.getLineSidePairFromCmLine(cmLine); DisplaySide side = pair.getSide(); if (cm.somethingSelected()) { // TODO: Handle range comment } else { insertNewDraft(side, cmLine); }
4989 1242678167GerritCodeReview/gerritEdwin Kempinf156a4d76a1521d9aeb8d029a5703e44cbef7b55 None TODO: Implement this SATD_ADDED setShowLineNumbers(boolean) void setShowLineNumbers(boolean b) // TODO: Implement this
4988 1242678166GerritCodeReview/gerritEdwin Kempinf156a4d76a1521d9aeb8d029a5703e44cbef7b55 None TODO: Handle showLineNumbers preference SATD_ADDED display(CommentsCollections) private void display(final CommentsCollections comments) final DiffPreferences prefs = getPrefs(); final DiffInfo diff = getDiff(); setThemeStyles(prefs.theme().isDark()); setShowIntraline(prefs.intralineDifference()); // TODO: Handle showLineNumbers preference cm = newCm(diff.metaA() == null ? diff.metaB() : diff.metaA(), diff.textUnified(), diffTable.cm); setShowTabs(prefs.showTabs()); chunkManager = new UnifiedChunkManager(this, cm, diffTable.scrollbar); skipManager = new UnifiedSkipManager(this, commentManager); operation(new Runnable() { @Override public void run() { // Estimate initial CM3 height, fixed up in onShowView. int height = Window.getClientHeight() - (Gerrit.getHeaderFooterHeight() + 18); cm.setHeight(height); render(diff); commentManager.render(comments, prefs.expandAllComments()); skipManager.render(prefs.context(), diff); } }); registerCmEvents(cm); setPrefsAction(new PreferencesAction(this, prefs)); header.init(getPrefsAction(), getSideBySideDiffLink(), diff.sideBySideWebLinks()); setAutoHideDiffHeader(prefs.autoHideDiffTableHeader()); setupSyntaxHighlighting();
4987 1242678165GerritCodeReview/gerritEdwin Kempinf156a4d76a1521d9aeb8d029a5703e44cbef7b55 None TODO skip may need to be handled SATD_ADDED textUnified() public final String textUnified() StringBuilder s = new StringBuilder(); JsArray c = content(); for (int i = 0; i < c.length(); i++) { Region r = c.get(i); if (r.ab() != null) { append(s, r.ab()); } else { if (r.a() != null) { append(s, r.a()); } if (r.b() != null) { append(s, r.b()); } } // TODO skip may need to be handled } return s.toString();
4986 1242678164GerritCodeReview/gerritEdwin Kempinf156a4d76a1521d9aeb8d029a5703e44cbef7b55 None TODO: verify addGutterTag SATD_ADDED initOnClick() private static final native JavaScriptObject initOnClick() mp
4985 1242678163GerritCodeReview/gerritEdwin Kempinf156a4d76a1521d9aeb8d029a5703e44cbef7b55 TODO: This is not optimal, but shouldn't be too costly in most cases. Maybe rewrite after done keeping track of diff chunk positions. TODO: This is not optimal, but shouldn't be too costly in most cases. Maybe rewrite after done keeping track of diff chunk positions. SATD_MOVED_FILE splitSkips(int, List) List splitSkips(int context, List skips) if (sideB.containsKey(0)) { // Special case of file comment; cannot skip first line. for (SkippedLine skip : skips) { if (skip.getStartB() == 0) { skip.incrementStart(1); } } } // TODO: This is not optimal, but shouldn't be too costly in most cases. // Maybe rewrite after done keeping track of diff chunk positions. for (int boxLine : sideB.tailMap(1).keySet()) { List temp = new ArrayList<>(skips.size() + 2); for (SkippedLine skip : skips) { int startLine = skip.getStartB(); int deltaBefore = boxLine - startLine; int deltaAfter = startLine + skip.getSize() - boxLine; if (deltaBefore < -context || deltaAfter < -context) { // Size guaranteed to be greater than 1 temp.add(skip); } else if (deltaBefore > context && deltaAfter > context) { SkippedLine before = new SkippedLine(skip.getStartA(), skip.getStartB(), skip.getSize() - deltaAfter - context); skip.incrementStart(deltaBefore + context); checkAndAddSkip(temp, before); checkAndAddSkip(temp, skip); } else if (deltaAfter > context) { skip.incrementStart(deltaBefore + context); checkAndAddSkip(temp, skip); } else if (deltaBefore > context) { skip.reduceSize(deltaAfter + context); checkAndAddSkip(temp, skip); } } if (temp.isEmpty()) { return temp; } skips = temp; } return skips;
4984 1242678162GerritCodeReview/gerritEdwin Kempinf156a4d76a1521d9aeb8d029a5703e44cbef7b55 Every comment appears in both side maps as a linked pair. It is only necessary to search one side to find a comment on either side of the editor pair. Every comment appears in both side maps as a linked pair. It is only necessary to search one side to find a comment on either side of the editor pair. SATD_MOVED_FILE commentNav(CodeMirror, Direction) return new Runnable() { @Override public void run() { // Every comment appears in both side maps as a linked pair. // It is only necessary to search one side to find a comment // on either side of the editor pair. SortedMap map = map(src.side()); int line = src.extras().hasActiveLine() ? src.getLineNumber(src.extras().activeLine()) + 1 : 0; if (dir == Direction.NEXT) { map = map.tailMap(line + 1); if (map.isEmpty()) { return; } line = map.firstKey(); } else { map = map.headMap(line); if (map.isEmpty()) { return; } line = map.lastKey(); } SideBySideCommentGroup g = map.get(line); if (g.getBoxCount() == 0) { g = g.getPeer(); } CodeMirror cm = g.getCm(); double y = cm.heightAtLine(g.getLine() - 1, "local"); cm.setCursor(Pos.create(g.getLine() - 1)); cm.scrollToY(y - 0.5 * cm.scrollbarV().getClientHeight()); cm.focus(); } }; Runnable commentNav(final CodeMirror src, final Direction dir)
4983 1242678161GerritCodeReview/gerritEdwin Kempinf156a4d76a1521d9aeb8d029a5703e44cbef7b55 First line workaround First line workaround SATD_MOVED_FILE collapse(int, int, boolean) void collapse(int start, int end, boolean attach) if (attach) { boolean isNew = lineWidget == null; Configuration cfg = Configuration.create().set("coverGutter", true).set("noHScroll", true); if (start == 0) { // First line workaround lineWidget = cm.addLineWidget(end + 1, getElement(), cfg.set("above", true)); } else { lineWidget = cm.addLineWidget(start - 1, getElement(), cfg); } if (isNew) { lineWidget.onFirstRedraw(new Runnable() { @Override public void run() { int w = cm.getGutterElement().getOffsetWidth(); getElement().getStyle().setPaddingLeft(w, Unit.PX); } }); } } textMarker = cm.markText(Pos.create(start, 0), Pos.create(end), Configuration.create().set("collapsed", true).set("inclusiveLeft", true).set("inclusiveRight", true)); textMarker.on("beforeCursorEnter", new Runnable() { @Override public void run() { expandAll(); } }); int skipped = end - start + 1; if (skipped <= UP_DOWN_THRESHOLD) { addStyleName(style.noExpand()); } else { upArrow.setHTML(PatchUtil.M.expandBefore(NUM_ROWS_TO_EXPAND)); downArrow.setHTML(PatchUtil.M.expandAfter(NUM_ROWS_TO_EXPAND)); } skipNum.setText(PatchUtil.M.patchSkipRegion(Integer.toString(skipped)));
4982 1242678160GerritCodeReview/gerritEdwin Kempinf156a4d76a1521d9aeb8d029a5703e44cbef7b55 First line workaround First line workaround SATD_MOVED_FILE collapse(int, int, boolean) void collapse(int start, int end, boolean attach) if (attach) { boolean isNew = lineWidget == null; Configuration cfg = Configuration.create().set("coverGutter", true).set("noHScroll", true); if (start == 0) { // First line workaround lineWidget = cm.addLineWidget(end + 1, getElement(), cfg.set("above", true)); } else { lineWidget = cm.addLineWidget(start - 1, getElement(), cfg); } if (isNew) { lineWidget.onFirstRedraw(new Runnable() { @Override public void run() { int w = cm.getGutterElement().getOffsetWidth(); getElement().getStyle().setPaddingLeft(w, Unit.PX); } }); } } textMarker = cm.markText(Pos.create(start, 0), Pos.create(end), Configuration.create().set("collapsed", true).set("inclusiveLeft", true).set("inclusiveRight", true)); textMarker.on("beforeCursorEnter", new Runnable() { @Override public void run() { expandAll(); } }); int skipped = end - start + 1; if (skipped <= UP_DOWN_THRESHOLD) { addStyleName(style.noExpand()); } else { upArrow.setHTML(PatchUtil.M.expandBefore(NUM_ROWS_TO_EXPAND)); downArrow.setHTML(PatchUtil.M.expandAfter(NUM_ROWS_TO_EXPAND)); } skipNum.setText(PatchUtil.M.patchSkipRegion(Integer.toString(skipped)));
4981 1242678159GerritCodeReview/gerritDave Borowitzed4d2358cf053eef762c7f517d7f1a1274e26de0 None Only until we improve ChangeRebuilder to call merge(). SATD_ADDED setSubmissionId(String) public void setSubmissionId(String submissionId) this.submissionId = submissionId;
4980 1242678158GerritCodeReview/gerritDave Borowitz561b8fad4336519b1a250d3a84e14a75f808bad0 None TODO(dborowitz): This should maybe be a synthetic timestamp just after the actual last update in the history. On the one hand using the commit updated time is reasonable, but on the other it might be non-monotonic, and who knows what would break then. SATD_ADDED uniquePerUpdate() boolean uniquePerUpdate() return true;
4979 1242678157GerritCodeReview/gerritDave Borowitz561b8fad4336519b1a250d3a84e14a75f808bad0 None Too much time elapsed. SATD_ADDED rebuildAsync(Change.Id, ListeningExecutorService) public ListenableFuture rebuildAsync(final Change.Id id, ListeningExecutorService executor) return executor.submit(new Callable() { @Override public Void call() throws Exception { try (ReviewDb db = schemaFactory.open()) { rebuild(db, id); } return null; } });
4976 1242678156GerritCodeReview/gerritDave Borowitzacda8b37256e1c2175fdebb78e700f345d1d14d4 None TODO: can go back to a list? SATD_ADDED create(ChangeControl, Date) mp ChangeDraftUpdate create(ChangeControl ctl, Date when)
4975 1242678155GerritCodeReview/gerritDave Borowitzacda8b37256e1c2175fdebb78e700f345d1d14d4 None Take advantage of an existing update on All-Users to prune any published comments from drafts. NoteDbUpdateManager takes care of ensuring that this update is applied before its dependent draft update. Deleting aggressively in this way, combined with filtering out duplicate published/draft comments in ChangeNotes#getDraftComments, makes up for the fact that updates between the change repo and All-Users are not atomic. TODO(dborowitz): We might want to distinguish between deleted drafts that we're fixing up after the fact by putting them in a separate commit. But note that we don't care much about the commit graph of the draft ref, particularly because the ref is completely deleted when all drafts are gone. SATD_ADDED checkComments(Map, Map) private void checkComments(Map existingNotes, Map toUpdate) throws OrmException // Prohibit various kinds of illegal operations on comments. Set existing = new HashSet<>(); for (RevisionNote rn : existingNotes.values()) { for (PatchLineComment c : rn.comments) { existing.add(c.getKey()); if (draftUpdate != null) { // Take advantage of an existing update on All-Users to prune any // published comments from drafts. NoteDbUpdateManager takes care of // ensuring that this update is applied before its dependent draft // update. // // Deleting aggressively in this way, combined with filtering out // duplicate published/draft comments in ChangeNotes#getDraftComments, // makes up for the fact that updates between the change repo and // All-Users are not atomic. // // TODO(dborowitz): We might want to distinguish between deleted // drafts that we're fixing up after the fact by putting them in a // separate commit. But note that we don't care much about the commit // graph of the draft ref, particularly because the ref is completely // deleted when all drafts are gone. draftUpdate.deleteComment(c.getRevId(), c.getKey()); } } } for (RevisionNoteBuilder b : toUpdate.values()) { for (PatchLineComment c : b.put.values()) { if (existing.contains(c.getKey())) { throw new OrmException("Cannot update existing published comment: " + c); } } }
4974 1242678005GerritCodeReview/gerritDave Borowitz103db472e649eaacf79ac0105088da7946bc53ae TODO(dborowitz): Smarter bucketing: pick a bucket start time T and include all events up to T + TS_WINDOW_MS but no further. Interleaving different authors complicates things. TODO(dborowitz): Smarter bucketing: pick a bucket start time T and include all events up to T + TS_WINDOW_MS but no further. Interleaving different authors complicates things. CLASS_OR_METHOD_CHANGED rebuildAsync(Change.Id, ListeningExecutorService) public ListenableFuture rebuildAsync(final Change.Id id, ListeningExecutorService executor) return executor.submit(new Callable() { @Override public Void call() throws Exception { try (ReviewDb db = schemaFactory.open()) { rebuild(db, id); } return null; } });
4972 1242678154GerritCodeReview/gerritUrs Wolfer0e8077dc8d69acaf80a8ee9c7a920c9603d9228a None TODO(dborowitz): share manager across changes. SATD_ADDED rebuildAsync(Change, ListeningExecutorService, Repository) public ListenableFuture rebuildAsync(final Change change, ListeningExecutorService executor, final Repository changeRepo) return executor.submit(new Callable() { @Override public Void call() throws Exception { rebuild(change, changeRepo); return null; } });
4971 1242678005GerritCodeReview/gerritUrs Wolfer0e8077dc8d69acaf80a8ee9c7a920c9603d9228a TODO(dborowitz): Smarter bucketing: pick a bucket start time T and include all events up to T + TS_WINDOW_MS but no further. Interleaving different authors complicates things. TODO(dborowitz): Smarter bucketing: pick a bucket start time T and include all events up to T + TS_WINDOW_MS but no further. Interleaving different authors complicates things. CLASS_OR_METHOD_CHANGED rebuildAsync(Change, ListeningExecutorService, Repository) public ListenableFuture rebuildAsync(final Change change, ListeningExecutorService executor, final Repository changeRepo) return executor.submit(new Callable() { @Override public Void call() throws Exception { rebuild(change, changeRepo); return null; } });
4970 1242677990GerritCodeReview/gerritDave Borowitz020c01372121efe820e63b9cf52cd4d9fa3d9a90 TODO - dborowitz: add NEW_CHANGE type for default. TODO - dborowitz: add NEW_CHANGE type for default. CLASS_OR_METHOD_CHANGED getChangeKindInternal(ChangeKindCache, ReviewDb, ChangeNotes, PatchSet, ChangeData.Factory, ProjectCache, GitRepositoryManager) private static ChangeKind getChangeKindInternal(ChangeKindCache cache, ReviewDb db, ChangeNotes notes, PatchSet patch, ChangeData.Factory changeDataFactory, ProjectCache projectCache, GitRepositoryManager repoManager) // TODO - dborowitz: add NEW_CHANGE type for default. ChangeKind kind = ChangeKind.REWORK; // Trivial case: if we're on the first patch, we don't need to open // the repository. if (patch.getId().get() > 1) { try (Repository repo = repoManager.openRepository(notes.getProjectName())) { ProjectState projectState = projectCache.checkedGet(notes.getProjectName()); ChangeData cd = changeDataFactory.create(db, notes); Collection patchSetCollection = cd.patchSets(); PatchSet priorPs = patch; for (PatchSet ps : patchSetCollection) { if (ps.getId().get() < patch.getId().get() && (ps.getId().get() > priorPs.getId().get() || priorPs == patch)) { // We only want the previous patch set, so walk until the last one priorPs = ps; } } // If we still think the previous patch is the current patch, // we only have one patch set. Return the default. // This can happen if a user creates a draft, uploads a second patch, // and deletes the draft. if (priorPs != patch) { kind = cache.getChangeKind(projectState, repo, ObjectId.fromString(priorPs.getRevision().get()), ObjectId.fromString(patch.getRevision().get())); } } catch (IOException | OrmException e) { // Do nothing; assume we have a complex change log.warn("Unable to get change kind for patchSet " + patch.getPatchSetId() + "of change " + notes.getChangeId(), e); } } return kind;
4969 1242678153GerritCodeReview/gerritDavid Pursehousec32abd0104868f6d798348fa74cff420124deb0e None TODO: The same method exists in EventFactory, find a common place for it SATD_ADDED toUserIdentity(PersonIdent) private UserIdentity toUserIdentity(final PersonIdent who) final UserIdentity u = new UserIdentity(); u.setName(who.getName()); u.setEmail(who.getEmailAddress()); u.setDate(new Timestamp(who.getWhen().getTime())); u.setTimeZone(who.getTimeZoneOffset()); // If only one account has access to this email address, select it // as the identity of the user. // final Set a = byEmailCache.get(u.getEmail()); if (a.size() == 1) { u.setAccount(a.iterator().next()); } return u;
4968 1242678152GerritCodeReview/gerritDavid Pursehousec32abd0104868f6d798348fa74cff420124deb0e None TODO: The same method exists in PatchSetInfoFactory, find a common place for it SATD_ADDED toUserIdentity(PersonIdent) private UserIdentity toUserIdentity(PersonIdent who) UserIdentity u = new UserIdentity(); u.setName(who.getName()); u.setEmail(who.getEmailAddress()); u.setDate(new Timestamp(who.getWhen().getTime())); u.setTimeZone(who.getTimeZoneOffset()); // If only one account has access to this email address, select it // as the identity of the user. // Set a = byEmailCache.get(u.getEmail()); if (a.size() == 1) { u.setAccount(a.iterator().next()); } return u;
4967 1242677990GerritCodeReview/gerritEdwin Kempin3ce9d7e6f4c58b79ddf85a29fcf03948a2e61a09 TODO - dborowitz: add NEW_CHANGE type for default. TODO - dborowitz: add NEW_CHANGE type for default. CLASS_OR_METHOD_CHANGED getChangeKindInternal(ChangeKindCache, ReviewDb, ChangeNotes, PatchSet, ChangeData.Factory, ProjectCache, GitRepositoryManager) private static ChangeKind getChangeKindInternal(ChangeKindCache cache, ReviewDb db, ChangeNotes notes, PatchSet patch, ChangeData.Factory changeDataFactory, ProjectCache projectCache, GitRepositoryManager repoManager) // TODO - dborowitz: add NEW_CHANGE type for default. ChangeKind kind = ChangeKind.REWORK; // Trivial case: if we're on the first patch, we don't need to open // the repository. if (patch.getId().get() > 1) { try (Repository repo = repoManager.openRepository(notes.getProjectName())) { ProjectState projectState = projectCache.checkedGet(notes.getProjectName()); ChangeData cd = changeDataFactory.create(db, notes); Collection patchSetCollection = cd.patchSets(); PatchSet priorPs = patch; for (PatchSet ps : patchSetCollection) { if (ps.getId().get() < patch.getId().get() && (ps.getId().get() > priorPs.getId().get() || priorPs == patch)) { // We only want the previous patch set, so walk until the last one priorPs = ps; } } // If we still think the previous patch is the current patch, // we only have one patch set. Return the default. // This can happen if a user creates a draft, uploads a second patch, // and deletes the draft. if (priorPs != patch) { kind = cache.getChangeKind(projectState, repo, ObjectId.fromString(priorPs.getRevision().get()), ObjectId.fromString(patch.getRevision().get())); } } catch (IOException | OrmException e) { // Do nothing; assume we have a complex change log.warn("Unable to get change kind for patchSet " + patch.getPatchSetId() + "of change " + notes.getChangeId(), e); } } return kind;
4966 1242678151GerritCodeReview/gerritDave Borowitz3e4b231afc5b87556ec9ec9affb7671f74eb5bd7 None better. SATD_ADDED asChanges(List) public static List asChanges(List changeDatas) throws OrmException List result = new ArrayList<>(changeDatas.size()); for (ChangeData cd : changeDatas) { result.add(cd.change()); } return result;
4965 1242678150GerritCodeReview/gerritDavid Ostrovsky81b830df4e37d398ad957bb979b039f8e13a73ab None ignore non valid numbers We don't want to popup another ugly dialog just to say \"The number you've provided is invalid, try again\" SATD_ADDED gotoLine() private Runnable gotoLine() return new Runnable() { @Override public void run() { String n = Window.prompt(EditConstants.I.gotoLineNumber(), ""); if (n != null) { try { int line = Integer.parseInt(n); line--; if (line >= 0) { cm.scrollToLine(line); } } catch (NumberFormatException e) { // ignore non valid numbers // We don't want to popup another ugly dialog just to say // "The number you've provided is invalid, try again" } } } };
4964 1242678149GerritCodeReview/gerritDavid Pursehouse5d8f62c938bb9e1e7d493ffe56159cc3f82fd098 None TODO(davido): Expose setting of account preferences in the API SATD_ADDED setSignedOffByFooter() private void setSignedOffByFooter() throws Exception RestResponse r = adminSession.get("/accounts/" + admin.email + "/preferences"); r.assertOK(); GeneralPreferencesInfo i = newGson().fromJson(r.getReader(), GeneralPreferencesInfo.class); i.signedOffBy = true; r = adminSession.put("/accounts/" + admin.email + "/preferences", i); r.assertOK(); GeneralPreferencesInfo o = newGson().fromJson(r.getReader(), GeneralPreferencesInfo.class); assertThat(o.signedOffBy).isTrue();
4963 1242678148GerritCodeReview/gerritDavid Pursehouseb06d2e33b2ee554744ad20626bc6df4cd3bd4cd7 None TODO(davido): Maintain cache of default values in AllUsers repository SATD_ADDED apply(ConfigResource) public GeneralPreferencesInfo apply(ConfigResource rsrc) throws IOException, ConfigInvalidException try (Repository git = gitMgr.openRepository(allUsersName)) { VersionedAccountPreferences p = VersionedAccountPreferences.forDefault(); p.load(git); GeneralPreferencesInfo a = new GeneralPreferencesInfo(); // TODO(davido): Maintain cache of default values in AllUsers repository return loader.loadFromAllUsers(a, p, git); }
4962 1242678147GerritCodeReview/gerritDave Borowitz01d4110dad2c2a9e5fdf2a9edb0343171b1d922f None TODO: Use InternalUser instead of change owner SATD_ADDED abandonAllOpenChangeForDeletedProject(Project.NameKey) private void abandonAllOpenChangeForDeletedProject(Project.NameKey destProject) try { for (ChangeData cd : internalChangeQuery.byProjectOpen(destProject)) { // TODO: Use InternalUser instead of change owner try (BatchUpdate bu = batchUpdateFactory.create(db, destProject, identifiedUserFactory.create(cd.change().getOwner()), TimeUtil.nowTs())) { bu.addOp(cd.getId(), new BatchUpdate.Op() { @Override public boolean updateChange(ChangeContext ctx) throws OrmException { Change change = ctx.getChange(); if (!change.getStatus().isOpen()) { return false; } change.setStatus(Change.Status.ABANDONED); ChangeMessage msg = new ChangeMessage(new ChangeMessage.Key(change.getId(), ChangeUtil.messageUUID(ctx.getDb())), null, change.getLastUpdatedOn(), change.currentPatchSetId()); msg.setMessage("Project was deleted."); cmUtil.addChangeMessage(ctx.getDb(), ctx.getUpdate(change.currentPatchSetId()), msg); ctx.saveChange(); return true; } }); try { bu.execute(); } catch (UpdateException | RestApiException e) { logWarn("Cannot abandon changes for deleted project " + destProject, e); } } } } catch (OrmException e) { logWarn("Cannot abandon changes for deleted project " + destProject, e); }
4958 1242678146GerritCodeReview/gerritEdwin Kempina1bfee927e684a05ef5a7f6fdbe34c3da3c06644 None TODO(ekempin): mark changes as merged inside of ReplaceOp SATD_ADDED get() public String get() return value;
4957 1242678145GerritCodeReview/gerritDavid Pursehouse3c377580a2dc8659a9c8b59567f52c0ca4a09ba8 None TODO(ekempin): Remove when database backend is deleted SATD_ADDED apply(PatchSetApproval) public Timestamp apply(PatchSetApproval input) return input.getGranted();
4956 1242678144GerritCodeReview/gerritEdwin Kempin72825f0f69cbeefbba1e6a7356e3e1778052dc8d None TODO(dborowitz): Remove when deleting index schemas <27. SATD_ADDED asChanges(List) public static List asChanges(List changeDatas) throws OrmException List result = new ArrayList<>(changeDatas.size()); for (ChangeData cd : changeDatas) { result.add(cd.change()); } return result;
4955 1242678143GerritCodeReview/gerritEdwin Kempin72825f0f69cbeefbba1e6a7356e3e1778052dc8d None TODO(dborowitz): Remove when deleting index schemas <27. SATD_ADDED apply(PatchSetApproval) public Timestamp apply(PatchSetApproval input) return input.getGranted();
4952 1242677917GerritCodeReview/gerritDave Borowitz8ea575a35bd343248dcf4efdc7583cd786ac1e30 TODO: propagate this exception TODO: propagate this exception CLASS_OR_METHOD_CHANGED controlFor(Project.NameKey, Change.Id, CurrentUser) public ChangeControl controlFor(Project.NameKey project, Change.Id changeId, CurrentUser user) throws NoSuchChangeException, OrmException return controlFor(notesFactory.create(db.get(), project, changeId), user);
4951 1242678142GerritCodeReview/gerritDave Borowitzfb0792d5a6f5da93bf74bc8ab91f88279efe6fd6 None If too many changes failed, maybe there was a bug in the indexer. Don't trust the results. This is not an exact percentage since we bump the same failure counter if a project can't be read, but close enough. SATD_ADDED success() public boolean success() return success;
4950 1242677917GerritCodeReview/gerritEdwin Kempin4b2384b5827fb9d1b315077f2a3ef3db668e3fcc TODO: propagate this exception TODO: propagate this exception CLASS_OR_METHOD_CHANGED controlFor(Project.NameKey, Change.Id, CurrentUser) public ChangeControl controlFor(Project.NameKey project, Change.Id changeId, CurrentUser user) throws NoSuchChangeException, OrmException return controlFor(notesFactory.create(db.get(), project, changeId), user);
4949 1242678141GerritCodeReview/gerritEdwin Kempin341baf22a0756b73ba36caec87fcb740061e332f None TODO Don't load the changes directly from the database, but provide project name + change ID to changeDataFactory, or delete this predicate. SATD_ADDED read() public ResultSet read() throws OrmException Set ids = new HashSet<>(); for (PatchLineComment sc : args.plcUtil.draftByAuthor(args.db.get(), accountId)) { ids.add(sc.getKey().getParentKey().getParentKey().getParentKey()); } List r = new ArrayList<>(ids.size()); // TODO Don't load the changes directly from the database, but provide // project name + change ID to changeDataFactory, or delete this predicate. for (Change c : args.db.get().changes().get(ids)) { r.add(args.changeDataFactory.create(args.db.get(), c)); } return new ListResultSet<>(r);
4948 1242678140GerritCodeReview/gerritEdwin Kempin341baf22a0756b73ba36caec87fcb740061e332f Just don't BCC everyone. Better to send a partial message to those we already have queued up then to fail deliver entirely to people who have a lower interest in the change. Just don't BCC everyone. Better to send a partial message to those we already have queued up then to fail deliver entirely to people who have a lower interest in the change. CLASS_OR_METHOD_CHANGED setFrom(Account.Id) public void setFrom(final Account.Id id) super.setFrom(id); /** * Is the from user in an email squelching group? */ final IdentifiedUser user = args.identifiedUserFactory.create(id); emailOnlyAuthors = !user.getCapabilities().canEmailReviewers();
4946 1242678139GerritCodeReview/gerritEdwin Kempin25b6287364f723d5bdca522d5dd83e98c558010a None TODO(ekempin): Pass project to changeDataFactory SATD_ADDED toChangeData(Document, Set, String) private ChangeData toChangeData(Document doc, Set fields, String idFieldName) ChangeData cd; // Either change or the ID field was guaranteed to be included in the call // to fields() above. BytesRef cb = doc.getBinaryValue(CHANGE_FIELD); if (cb != null) { cd = changeDataFactory.create(db.get(), ChangeProtoField.CODEC.decode(cb.bytes, cb.offset, cb.length)); } else { int id = doc.getField(idFieldName).numericValue().intValue(); // TODO(ekempin): Pass project to changeDataFactory @SuppressWarnings("unused") Project.NameKey project = new Project.NameKey(doc.getField(PROJECT.getName()).stringValue()); cd = changeDataFactory.create(db.get(), new Change.Id(id)); } if (fields.contains(PATCH_SET_FIELD)) { decodePatchSets(doc, cd); } if (fields.contains(APPROVAL_FIELD)) { decodeApprovals(doc, cd); } if (fields.contains(ADDED_FIELD) && fields.contains(DELETED_FIELD)) { decodeChangedLines(doc, cd); } if (fields.contains(MERGEABLE_FIELD)) { decodeMergeable(doc, cd); } if (fields.contains(REVIEWEDBY_FIELD)) { decodeReviewedBy(doc, cd); } return cd;
4945 1242678138GerritCodeReview/gerritEdwin Kempinc89c21cd6d0537e4e81807fbb93da07905d64f0b None TODO: Throw NoSuchChangeException when the change is not found in the database SATD_ADDED apply(PatchSetApproval) public Timestamp apply(PatchSetApproval input) return input.getGranted();
4944 1242678134GerritCodeReview/gerritEdwin Kempinc89c21cd6d0537e4e81807fbb93da07905d64f0b TODO(dborowitz): Shouldn't have to look up Change. TODO(dborowitz): Reuse open repository from caller. SATD_CHANGED create(Multimap, ReviewDb, PatchSetUtil, ChangeNotes.Factory, Project.NameKey) public static GroupCollector create(Multimap changeRefsById, final ReviewDb db, final PatchSetUtil psUtil, final ChangeNotes.Factory notesFactory, final Project.NameKey project) return new GroupCollector(transformRefs(changeRefsById), new Lookup() { @Override public List lookup(PatchSet.Id psId) throws OrmException { // TODO(dborowitz): Reuse open repository from caller. ChangeNotes notes = notesFactory.create(db, project, psId.getParentKey()); PatchSet ps = psUtil.get(db, notes, psId); return ps != null ? ps.getGroups() : null; } });
4943 1242678137GerritCodeReview/gerritDave Borowitzdfbb9915cd727799ed4fda483214b628de048e72 None TODO(dborowitz): Seems like this could get expensive for many patch sets. Is there a more efficient implementation? SATD_ADDED findAnyMergedInto(CodeReviewRevWalk, Iterable, CodeReviewCommit) public static CodeReviewCommit findAnyMergedInto(CodeReviewRevWalk rw, Iterable commits, CodeReviewCommit tip) throws IOException for (CodeReviewCommit c : commits) { // TODO(dborowitz): Seems like this could get expensive for many patch // sets. Is there a more efficient implementation? if (rw.isMergedInto(c, tip)) { return c; } } return null;
4942 1242678136GerritCodeReview/gerritDave Borowitzdfbb9915cd727799ed4fda483214b628de048e72 None No patch set for the already merged commit, although we know it came form a patch set ref. Fix up the database. Note that this uses the current user as the uploader, which is as good a guess as any. SATD_ADDED getOrCreateAlreadyMergedPatchSet(ChangeContext) private PatchSet getOrCreateAlreadyMergedPatchSet(ChangeContext ctx) throws IOException, OrmException PatchSet.Id psId = alreadyMerged.getPatchsetId(); logDebug("Fixing up already-merged patch set {}", psId); PatchSet prevPs = args.psUtil.current(ctx.getDb(), ctx.getNotes()); ctx.getRevWalk().parseBody(alreadyMerged); ctx.getChange().setCurrentPatchSet(psId, alreadyMerged.getShortMessage(), ctx.getChange().getOriginalSubject()); PatchSet existing = args.psUtil.get(ctx.getDb(), ctx.getNotes(), psId); if (existing != null) { logDebug("Patch set row exists, only updating change"); return existing; } // No patch set for the already merged commit, although we know it came form // a patch set ref. Fix up the database. Note that this uses the current // user as the uploader, which is as good a guess as any. List groups = prevPs != null ? prevPs.getGroups() : GroupCollector.getDefaultGroups(alreadyMerged); return args.psUtil.insert(ctx.getDb(), ctx.getRevWalk(), ctx.getUpdate(psId), psId, alreadyMerged, false, groups, null);
4941 1242678135GerritCodeReview/gerritEdwin Kempin82088ee913a63723cc779f3ee78f22646919ddbd None we cannot reconstruct the submit records for when this change was submitted, this is why we must fix the status SATD_ADDED get() public String get() return value;
4940 1242678134GerritCodeReview/gerritDave Borowitz39782443e9a52e134b2c501f4b9c6e68ffae8937 None TODO(dborowitz): Shouldn't have to look up Change. SATD_ADDED create(Multimap, ReviewDb, PatchSetUtil, ChangeNotes.Factory) public static GroupCollector create(Multimap changeRefsById, final ReviewDb db, final PatchSetUtil psUtil, final ChangeNotes.Factory notesFactory) return new GroupCollector(transformRefs(changeRefsById), new Lookup() { @Override public List lookup(PatchSet.Id psId) throws OrmException { // TODO(dborowitz): Shouldn't have to look up Change. Change c = db.changes().get(psId.getParentKey()); if (c == null) { return null; } ChangeNotes notes = notesFactory.create(c); PatchSet ps = psUtil.get(db, notes, psId); return ps != null ? ps.getGroups() : null; } });
4938 1242678133GerritCodeReview/gerritDave Borowitzf10ac66574b05ce49ee44ce889a01e5f1461e714 None TODO(dborowitz): Re-enable when ConsistencyChecker supports notedb. SATD_ADDED check() public void check() throws Exception // TODO(dborowitz): Re-enable when ConsistencyChecker supports notedb. assume().that(notesMigration.enabled()).isFalse(); PushOneCommit.Result r = createChange(); assertThat(gApi.changes().id(r.getChangeId()).get().problems).isNull(); assertThat(gApi.changes().id(r.getChangeId()).get(EnumSet.of(ListChangesOption.CHECK)).problems).isEmpty();
4934 1242678132GerritCodeReview/gerritEdwin Kempin2a85fd572e0c2afdef5ac9dd7267954cc5ac46b9 None TODO(dborowitz): Stamp approximate approvals at this time. SATD_ADDED apply(ChangeUpdate) void apply(ChangeUpdate update) throws OrmException if (!Objects.equals(change.getTopic(), notedbChange.getTopic())) { update.setTopic(change.getTopic()); } if (!Objects.equals(change.getStatus(), notedbChange.getStatus())) { // TODO(dborowitz): Stamp approximate approvals at this time. update.fixStatus(change.getStatus()); } if (!update.isEmpty()) { update.setSubjectForCommit("Final notedb migration updates"); }
4933 1242678131GerritCodeReview/gerritDave Borowitzd5db7d0ed0f1d0e393f38af04677655400cfc91f None TODO(dborowitz): Re-enable when ConsistencyChecker supports notedb. SATD_ADDED setUp() public void setUp() throws Exception // TODO(dborowitz): Re-enable when ConsistencyChecker supports notedb. assume().that(notesMigration.enabled()).isFalse(); // Ignore client clone of project; repurpose as server-side TestRepository. testRepo = new TestRepository<>((InMemoryRepository) repoManager.openRepository(project)); tip = testRepo.getRevWalk().parseCommit(testRepo.getRepository().exactRef("HEAD").getObjectId()); adminId = admin.getId(); checker = checkerProvider.get();
4930 1242678125GerritCodeReview/gerritEdwin Kempin12fbf41e95c3cd46fc2a8986a719c3f3cd4d742c TODO(dborowitz): Write to notedb. TODO(dborowitz): Write to notedb. CLASS_OR_METHOD_CHANGED insert(ReviewDb, ChangeUpdate, PatchSet.Id, RevCommit, boolean, Iterable, String) public PatchSet insert(ReviewDb db, ChangeUpdate update, PatchSet.Id psId, RevCommit commit, boolean draft, Iterable groups, String pushCertificate) throws OrmException Change.Id changeId = update.getChange().getId(); checkArgument(psId.getParentKey().equals(changeId), "cannot insert patch set %s on change %s", psId, changeId); if (update.getPatchSetId() != null) { checkArgument(update.getPatchSetId().equals(psId), "cannot insert patch set %s on update for %s", psId, update.getPatchSetId()); } else { update.setPatchSetId(psId); } PatchSet ps = new PatchSet(psId); ps.setRevision(new RevId(commit.name())); ps.setUploader(update.getUser().getAccountId()); ps.setCreatedOn(new Timestamp(update.getWhen().getTime())); ps.setDraft(draft); ps.setGroups(groups); ps.setPushCertificate(pushCertificate); db.patchSets().insert(Collections.singleton(ps)); if (!update.getChange().getSubject().equals(commit.getShortMessage())) { update.setSubject(commit.getShortMessage()); } if (migration.writeChanges()) { // TODO(dborowitz): Write to notedb. } return ps;
4927 1242678129GerritCodeReview/gerritDave Borowitz8fac6e1cbf6420079e8d76132abf3358ff471b93 None TODO(dborowitz): Convert to BatchUpdate. SATD_ADDED abandonOneChange(Change) private void abandonOneChange(Change change) throws OrmException, NoSuchChangeException, IOException db.changes().beginTransaction(change.getId()); // TODO(dborowitz): support InternalUser in ChangeUpdate ChangeControl control = changeControlFactory.controlFor(change, identifiedUserFactory.create(change.getOwner())); // TODO(dborowitz): Convert to BatchUpdate. ChangeUpdate update = changeUpdateFactory.create(control); try { change = db.changes().atomicUpdate(change.getId(), new AtomicUpdate() { @Override public Change update(Change change) { if (change.getStatus().isOpen()) { change.setStatus(Change.Status.ABANDONED); return change; } return null; } }); if (change != null) { ChangeMessage msg = new ChangeMessage(new ChangeMessage.Key(change.getId(), ChangeUtil.messageUUID(db)), null, change.getLastUpdatedOn(), change.currentPatchSetId()); msg.setMessage("Project was deleted."); // TODO(yyonas): atomic change is not propagated. cmUtil.addChangeMessage(db, update, msg); db.commit(); indexer.index(db, change); } } finally { db.rollback(); } update.commit();
4926 1242678128GerritCodeReview/gerritDave Borowitz8fac6e1cbf6420079e8d76132abf3358ff471b93 None TODO(dborowitz): Use PatchSetUtil? But we don't have a recent notes. SATD_ADDED setMerged(ChangeContext, ChangeMessage) private void setMerged(ChangeContext ctx, ChangeMessage msg) throws OrmException Change c = ctx.getChange(); ReviewDb db = ctx.getDb(); logDebug("Setting change {} merged", c.getId()); // TODO(dborowitz): Use PatchSetUtil? But we don't have a recent notes. mergedPatchSet = db.patchSets().get(c.currentPatchSetId()); c.setStatus(Change.Status.MERGED); c.setSubmissionId(args.submissionId); ctx.saveChange(); // TODO(dborowitz): We need to be able to change the author of the message, // which is not the user from the update context. addMergedMessage was able // to do this in the past. if (msg != null) { args.cmUtil.addChangeMessage(db, ctx.getUpdate(msg.getPatchSetId()), msg); }
4925 1242678127GerritCodeReview/gerritDave Borowitz8fac6e1cbf6420079e8d76132abf3358ff471b93 None TODO(dborowitz): Pre-BatchUpdate behavior wrote the merged message on the old patch set ID, so that's what we do here. I don't think this was intentional, and it should be changed. SATD_ADDED message(ChangeContext, String) private ChangeMessage message(ChangeContext ctx, String body) String uuid; try { uuid = ChangeUtil.messageUUID(ctx.getDb()); } catch (OrmException e) { return null; } ChangeMessage m = new ChangeMessage(new ChangeMessage.Key(ctx.getChange().getId(), uuid), // TODO(dborowitz): Pre-BatchUpdate behavior wrote the merged message on // the old patch set ID, so that's what we do here. I don't think this // was intentional, and it should be changed. null, ctx.getWhen(), toMerge.change().currentPatchSetId()); m.setMessage(body); return m;
4924 1242678117GerritCodeReview/gerritDave Borowitz8fac6e1cbf6420079e8d76132abf3358ff471b93 TODO(dborowitz): Can't use repo from ctx due to canMergeFlag. TODO(dborowitz): Can't use repo from ctx due to canMergeFlag. CLASS_OR_METHOD_CHANGED updateRepoImpl(RepoContext) public void updateRepoImpl(RepoContext ctx) throws IntegrationException, IOException // There are multiple parents, so this is a merge commit. We don't want // to rebase the merge as clients can't easily rebase their history with // that merge present and replaced by an equivalent merge with a different // first parent. So instead behave as though MERGE_IF_NECESSARY was // configured. MergeTip mergeTip = args.mergeTip; if (args.rw.isMergedInto(mergeTip.getCurrentTip(), toMerge)) { mergeTip.moveTipTo(toMerge, toMerge); acceptMergeTip(mergeTip); } else { // TODO(dborowitz): Can't use repo from ctx due to canMergeFlag. CodeReviewCommit newTip = args.mergeUtil.mergeOneCommit(args.serverIdent, args.serverIdent, args.repo, args.rw, args.inserter, args.canMergeFlag, args.destBranch, mergeTip.getCurrentTip(), toMerge); mergeTip.moveTipTo(newTip, toMerge); } args.mergeUtil.markCleanMerges(args.rw, args.canMergeFlag, mergeTip.getCurrentTip(), args.alreadyAccepted); acceptMergeTip(mergeTip);
4920 1242678126GerritCodeReview/gerritDave Borowitzcc1dfc5282c9d78ccc7f878cef6a31604e668947 None TODO(dborowitz): make non-public when converting to BatchUpdate. SATD_ADDED module() public static Module module() return new FactoryModule() { @Override protected void configure() { factory(SubmitStrategy.Arguments.Factory.class); } };
4918 1242678125GerritCodeReview/gerritDave Borowitz94ed8fd25832443b9c13710b0fdb29b1275a64d3 None TODO(dborowitz): Write to notedb. SATD_ADDED insert(ReviewDb, ChangeUpdate, PatchSet.Id, ObjectId, boolean, Iterable, String) public PatchSet insert(ReviewDb db, ChangeUpdate update, PatchSet.Id psId, ObjectId id, boolean draft, Iterable groups, String pushCertificate) throws OrmException Change.Id changeId = update.getChange().getId(); checkArgument(psId.getParentKey().equals(changeId), "cannot insert patch set %s on change %s", psId, changeId); if (update.getPatchSetId() != null) { checkArgument(update.getPatchSetId().equals(psId), "cannot insert patch set %s on update for %s", psId, update.getPatchSetId()); } else { update.setPatchSetId(psId); } PatchSet ps = new PatchSet(psId); ps.setRevision(new RevId(id.name())); ps.setUploader(update.getUser().getAccountId()); ps.setCreatedOn(new Timestamp(update.getWhen().getTime())); ps.setDraft(draft); ps.setGroups(groups); ps.setPushCertificate(pushCertificate); db.patchSets().insert(Collections.singleton(ps)); if (migration.writeChanges()) { // TODO(dborowitz): Write to notedb. } return ps;
4917 1242678124GerritCodeReview/gerritDave Borowitz3ecfd766745eba6ef59324de29c86e6c66b289ab None TODO(dborowitz): Use PatchSetUtil after BatchUpdate migration. SATD_ADDED setMerged(Change, ChangeMessage, ObjectId) private void setMerged(Change c, ChangeMessage msg, ObjectId mergeResultRev) throws OrmException, IOException logDebug("Setting change {} merged", c.getId()); ChangeUpdate update = null; final PatchSetApproval submitter; PatchSet merged; try { db.changes().beginTransaction(c.getId()); // We must pull the patchset out of commits, because the patchset ID is // modified when using the cherry-pick merge strategy. CodeReviewCommit commit = commits.get(c.getId()); PatchSet.Id mergedId = commit.change().currentPatchSetId(); // TODO(dborowitz): Use PatchSetUtil after BatchUpdate migration. merged = db.patchSets().get(mergedId); c = setMergedPatchSet(c.getId(), mergedId); submitter = approvalsUtil.getSubmitter(db, commit.notes(), mergedId); ChangeControl control = commit.getControl(); update = updateFactory.create(control, c.getLastUpdatedOn()); // TODO(yyonas): we need to be able to change the author of the message // is not the person for whom the change was made. addMergedMessage // did this in the past. if (msg != null) { cmUtil.addChangeMessage(db, update, msg); } db.commit(); } finally { db.rollback(); } update.commit(); indexer.index(db, c); try { mergedSenderFactory.create(c.getId(), submitter != null ? submitter.getAccountId() : null).sendAsync(); } catch (Exception e) { log.error("Cannot email merged notification for " + c.getId(), e); } if (submitter != null && mergeResultRev != null) { try { hooks.doChangeMergedHook(c, accountCache.get(submitter.getAccountId()).getAccount(), merged, db, mergeResultRev.name()); } catch (OrmException ex) { logError("Cannot run hook for submitted patch set " + c.getId(), ex); } }
4916 1242678123GerritCodeReview/gerritDave Borowitz3ecfd766745eba6ef59324de29c86e6c66b289ab None TODO(dborowitz): Read from notedb. SATD_ADDED get(ReviewDb, ChangeNotes, PatchSet.Id) public PatchSet get(ReviewDb db, ChangeNotes notes, PatchSet.Id psId) throws OrmException return db.patchSets().get(psId);
4915 1242678122GerritCodeReview/gerritDave Borowitz3ecfd766745eba6ef59324de29c86e6c66b289ab None TODO(dborowitz): PatchSetUtil. SATD_ADDED lookup(PatchSet.Id) public List lookup(PatchSet.Id psId) throws OrmException // TODO(dborowitz): PatchSetUtil. PatchSet ps = db.patchSets().get(psId); return ps != null ? ps.getGroups() : null;
4914 1242677792GerritCodeReview/gerritDave Borowitz3ecfd766745eba6ef59324de29c86e6c66b289ab This seems like a cheap trick. It doesn't properly account for a file that gets renamed between patch set 1 and patch set 2. We will wind up packing the wrong Patch object because we didn't do proper rename detection between the patch sets. This seems like a cheap trick. It doesn't properly account for a file that gets renamed between patch set 1 and patch set 2. We will wind up packing the wrong Patch object because we didn't do proper rename detection between the patch sets. CLASS_OR_METHOD_CHANGED loadCommentsAndHistory(ChangeNotes, ChangeType, String, String) private void loadCommentsAndHistory(ChangeNotes notes, ChangeType changeType, String oldName, String newName) throws OrmException Map byKey = new HashMap<>(); if (loadHistory) { // This seems like a cheap trick. It doesn't properly account for a // file that gets renamed between patch set 1 and patch set 2. We // will wind up packing the wrong Patch object because we didn't do // proper rename detection between the patch sets. // history = new ArrayList<>(); for (PatchSet ps : psUtil.byChange(db, notes)) { if (!control.isPatchVisible(ps, db)) { continue; } String name = fileName; if (psa != null) { switch(changeType) { case COPIED: case RENAMED: if (ps.getId().equals(psa)) { name = oldName; } break; case MODIFIED: case DELETED: case ADDED: case REWRITE: break; } } Patch p = new Patch(new Patch.Key(ps.getId(), name)); history.add(p); byKey.put(p.getKey(), p); } if (edit != null && edit.isPresent()) { Patch p = new Patch(new Patch.Key(new PatchSet.Id(psb.getParentKey(), 0), fileName)); history.add(p); byKey.put(p.getKey(), p); } } if (loadComments && edit == null) { AccountInfoCacheFactory aic = aicFactory.create(); comments = new CommentDetail(psa, psb); switch(changeType) { case ADDED: case MODIFIED: loadPublished(byKey, aic, newName); break; case DELETED: loadPublished(byKey, aic, newName); break; case COPIED: case RENAMED: if (psa != null) { loadPublished(byKey, aic, oldName); } loadPublished(byKey, aic, newName); break; case REWRITE: break; } CurrentUser user = control.getUser(); if (user.isIdentifiedUser()) { Account.Id me = user.getAccountId(); switch(changeType) { case ADDED: case MODIFIED: loadDrafts(byKey, aic, me, newName); break; case DELETED: loadDrafts(byKey, aic, me, newName); break; case COPIED: case RENAMED: if (psa != null) { loadDrafts(byKey, aic, me, oldName); } loadDrafts(byKey, aic, me, newName); break; case REWRITE: break; } } comments.setAccountInfoCache(aic.create()); }
4913 1242678121GerritCodeReview/gerritDavid Ostrovskyd287ab744d4d724fa68eca1ae4a68c833d71a1ed None TODO(davido): accept closure passed in from caller SATD_ADDED loadSection(Config, String, String, T, T, T) public static T loadSection(Config cfg, String section, String sub, T s, T defaults, T i) throws ConfigInvalidException try { for (Field f : s.getClass().getDeclaredFields()) { if (skipField(f)) { continue; } Class t = f.getType(); String n = f.getName(); f.setAccessible(true); Object d = f.get(defaults); if (!isString(t) && !isCollectionOrMap(t)) { Preconditions.checkNotNull(d, "Default cannot be null for: " + n); } if (isString(t)) { String v = cfg.getString(section, sub, n); if (v == null) { v = (String) d; } f.set(s, v); } else if (isInteger(t)) { f.set(s, cfg.getInt(section, sub, n, (Integer) d)); } else if (isLong(t)) { f.set(s, cfg.getLong(section, sub, n, (Long) d)); } else if (isBoolean(t)) { boolean b = cfg.getBoolean(section, sub, n, (Boolean) d); if (b || i != null) { f.set(s, b); } } else if (t.isEnum()) { f.set(s, cfg.getEnum(section, sub, n, (Enum) d)); } else if (isCollectionOrMap(t)) { // TODO(davido): accept closure passed in from caller continue; } else { throw new ConfigInvalidException("type is unknown: " + t.getName()); } if (i != null) { Object o = f.get(i); if (o != null) { f.set(s, o); } } } } catch (SecurityException | IllegalArgumentException | IllegalAccessException e) { throw new ConfigInvalidException("cannot load values", e); } return s;
4912 1242678120GerritCodeReview/gerritDavid Ostrovskyd287ab744d4d724fa68eca1ae4a68c833d71a1ed None TODO(davido): accept closure passed in from caller SATD_ADDED storeSection(Config, String, String, T, T) public static void storeSection(Config cfg, String section, String sub, T s, T defaults) throws ConfigInvalidException try { for (Field f : s.getClass().getDeclaredFields()) { if (skipField(f)) { continue; } Class t = f.getType(); String n = f.getName(); f.setAccessible(true); Object c = f.get(s); Object d = f.get(defaults); if (!isString(t) && !isCollectionOrMap(t)) { Preconditions.checkNotNull(d, "Default cannot be null for: " + n); } if (c == null || c.equals(d)) { cfg.unset(section, sub, n); } else { if (isString(t)) { cfg.setString(section, sub, n, (String) c); } else if (isInteger(t)) { cfg.setInt(section, sub, n, (Integer) c); } else if (isLong(t)) { cfg.setLong(section, sub, n, (Long) c); } else if (isBoolean(t)) { cfg.setBoolean(section, sub, n, (Boolean) c); } else if (t.isEnum()) { cfg.setEnum(section, sub, n, (Enum) c); } else if (isCollectionOrMap(t)) { // TODO(davido): accept closure passed in from caller continue; } else { throw new ConfigInvalidException("type is unknown: " + t.getName()); } } } } catch (SecurityException | IllegalArgumentException | IllegalAccessException e) { throw new ConfigInvalidException("cannot save values", e); }
4911 1242678119GerritCodeReview/gerritDave Borowitz82253c6e514a9ac546e5305534859d15e28ad188 TODO(dborowitz): Consider putting ChangeData in CodeReviewCommit. TODO(dborowitz): Consider putting ChangeData in CodeReviewCommit. CLASS_OR_METHOD_CHANGED validateChangeList(OpenRepo, Collection) private BranchBatch validateChangeList(OpenRepo or, Collection submitted) throws IntegrationException logDebug("Validating {} changes", submitted.size()); List toSubmit = new ArrayList<>(submitted.size()); Multimap revisions = getRevisions(or, submitted); SubmitType submitType = null; ChangeData choseSubmitTypeFrom = null; for (ChangeData cd : submitted) { Change.Id changeId = cd.getId(); ChangeControl ctl; Change chg; try { ctl = cd.changeControl(); chg = cd.change(); } catch (OrmException e) { commits.logProblem(changeId, e); continue; } if (chg.currentPatchSetId() == null) { String msg = "Missing current patch set on change"; logError(msg + " " + changeId); commits.problem(changeId, msg); continue; } PatchSet ps; Branch.NameKey destBranch = chg.getDest(); try { ps = cd.currentPatchSet(); } catch (OrmException e) { commits.logProblem(changeId, e); continue; } if (ps == null || ps.getRevision() == null || ps.getRevision().get() == null) { commits.logProblem(changeId, "Missing patch set or revision on change"); continue; } String idstr = ps.getRevision().get(); ObjectId id; try { id = ObjectId.fromString(idstr); } catch (IllegalArgumentException e) { commits.logProblem(changeId, e); continue; } if (!revisions.containsEntry(id, ps.getId())) { // TODO this is actually an error, the branch is gone but we // want to merge the issue. We can't safely do that if the // tip is not reachable. // commits.logProblem(changeId, "Revision " + idstr + " of patch set " + ps.getPatchSetId() + " does not match " + ps.getId().toRefName() + " for change"); continue; } CodeReviewCommit commit; try { commit = or.rw.parseCommit(id); } catch (IOException e) { commits.logProblem(changeId, e); continue; } // TODO(dborowitz): Consider putting ChangeData in CodeReviewCommit. commit.setControl(ctl); commit.setPatchsetId(ps.getId()); commits.put(commit); MergeValidators mergeValidators = mergeValidatorsFactory.create(); try { mergeValidators.validatePreMerge(or.repo, commit, or.project, destBranch, ps.getId(), caller); } catch (MergeValidationException mve) { commits.problem(changeId, mve.getMessage()); continue; } SubmitType st = getSubmitType(cd); if (st == null) { commits.logProblem(changeId, "No submit type for change"); continue; } if (submitType == null) { submitType = st; choseSubmitTypeFrom = cd; } else if (st != submitType) { commits.problem(changeId, String.format("Change has submit type %s, but previously chose submit type %s " + "from change %s in the same batch", st, submitType, choseSubmitTypeFrom.getId())); continue; } commit.add(or.canMergeFlag); toSubmit.add(cd); } logDebug("Submitting on this run: {}", toSubmit); return new AutoValue_MergeOp_BranchBatch(submitType, toSubmit);
4910 1242678118GerritCodeReview/gerritDave Borowitz82253c6e514a9ac546e5305534859d15e28ad188 TODO this is actually an error, the branch is gone but we want to merge the issue. We can't safely do that if the tip is not reachable. TODO this is actually an error, the branch is gone but we want to merge the issue. We can't safely do that if the tip is not reachable. CLASS_OR_METHOD_CHANGED validateChangeList(OpenRepo, Collection) private BranchBatch validateChangeList(OpenRepo or, Collection submitted) throws IntegrationException logDebug("Validating {} changes", submitted.size()); List toSubmit = new ArrayList<>(submitted.size()); Multimap revisions = getRevisions(or, submitted); SubmitType submitType = null; ChangeData choseSubmitTypeFrom = null; for (ChangeData cd : submitted) { Change.Id changeId = cd.getId(); ChangeControl ctl; Change chg; try { ctl = cd.changeControl(); chg = cd.change(); } catch (OrmException e) { commits.logProblem(changeId, e); continue; } if (chg.currentPatchSetId() == null) { String msg = "Missing current patch set on change"; logError(msg + " " + changeId); commits.problem(changeId, msg); continue; } PatchSet ps; Branch.NameKey destBranch = chg.getDest(); try { ps = cd.currentPatchSet(); } catch (OrmException e) { commits.logProblem(changeId, e); continue; } if (ps == null || ps.getRevision() == null || ps.getRevision().get() == null) { commits.logProblem(changeId, "Missing patch set or revision on change"); continue; } String idstr = ps.getRevision().get(); ObjectId id; try { id = ObjectId.fromString(idstr); } catch (IllegalArgumentException e) { commits.logProblem(changeId, e); continue; } if (!revisions.containsEntry(id, ps.getId())) { // TODO this is actually an error, the branch is gone but we // want to merge the issue. We can't safely do that if the // tip is not reachable. // commits.logProblem(changeId, "Revision " + idstr + " of patch set " + ps.getPatchSetId() + " does not match " + ps.getId().toRefName() + " for change"); continue; } CodeReviewCommit commit; try { commit = or.rw.parseCommit(id); } catch (IOException e) { commits.logProblem(changeId, e); continue; } // TODO(dborowitz): Consider putting ChangeData in CodeReviewCommit. commit.setControl(ctl); commit.setPatchsetId(ps.getId()); commits.put(commit); MergeValidators mergeValidators = mergeValidatorsFactory.create(); try { mergeValidators.validatePreMerge(or.repo, commit, or.project, destBranch, ps.getId(), caller); } catch (MergeValidationException mve) { commits.problem(changeId, mve.getMessage()); continue; } SubmitType st = getSubmitType(cd); if (st == null) { commits.logProblem(changeId, "No submit type for change"); continue; } if (submitType == null) { submitType = st; choseSubmitTypeFrom = cd; } else if (st != submitType) { commits.problem(changeId, String.format("Change has submit type %s, but previously chose submit type %s " + "from change %s in the same batch", st, submitType, choseSubmitTypeFrom.getId())); continue; } commit.add(or.canMergeFlag); toSubmit.add(cd); } logDebug("Submitting on this run: {}", toSubmit); return new AutoValue_MergeOp_BranchBatch(submitType, toSubmit);
4909 1242678032GerritCodeReview/gerritDavid Pursehouse84bc19bc3d537601a7df5be670d977b113821344 TODO(dborowitz): Consider putting ChangeData in CodeReviewCommit. TODO(dborowitz): Consider putting ChangeData in CodeReviewCommit. CLASS_OR_METHOD_CHANGED ChangeData> validateChangeList(Collection, IdentifiedUser) private ListMultimap validateChangeList(Collection submitted, IdentifiedUser caller) throws IntegrationException logDebug("Validating {} changes", submitted.size()); ListMultimap toSubmit = ArrayListMultimap.create(); Map allRefs; try { allRefs = repo.getRefDatabase().getRefs(ALL); } catch (IOException e) { throw new IntegrationException(e.getMessage(), e); } Set tips = new HashSet<>(); for (Ref r : allRefs.values()) { tips.add(r.getObjectId()); } for (ChangeData cd : submitted) { ChangeControl ctl; Change chg; try { ctl = cd.changeControl(); // Reload change in case index was stale. chg = cd.reloadChange(); } catch (OrmException e) { throw new IntegrationException("Failed to validate changes", e); } Change.Id changeId = cd.getId(); if (chg.getStatus() != Change.Status.NEW) { logDebug("Change {} is not new: {}", changeId, chg.getStatus()); continue; } if (chg.currentPatchSetId() == null) { logError("Missing current patch set on change " + changeId); commits.put(changeId, CodeReviewCommit.noPatchSet(ctl)); continue; } PatchSet ps; Branch.NameKey destBranch = chg.getDest(); try { ps = cd.currentPatchSet(); } catch (OrmException e) { throw new IntegrationException("Cannot query the database", e); } if (ps == null || ps.getRevision() == null || ps.getRevision().get() == null) { logError("Missing patch set or revision on change " + changeId); commits.put(changeId, CodeReviewCommit.noPatchSet(ctl)); continue; } String idstr = ps.getRevision().get(); ObjectId id; try { id = ObjectId.fromString(idstr); } catch (IllegalArgumentException iae) { logError("Invalid revision on patch set " + ps.getId()); commits.put(changeId, CodeReviewCommit.noPatchSet(ctl)); continue; } if (!tips.contains(id)) { // TODO Technically the proper way to do this test is to use a // RevWalk on "$id --not --all" and test for an empty set. But // that is way slower than looking for a ref directly pointing // at the desired tip. We should always have a ref available. // // TODO this is actually an error, the branch is gone but we // want to merge the issue. We can't safely do that if the // tip is not reachable. // logError("Revision " + idstr + " of patch set " + ps.getId() + " is not contained in any ref"); commits.put(changeId, CodeReviewCommit.revisionGone(ctl)); continue; } CodeReviewCommit commit; try { commit = rw.parseCommit(id); } catch (IOException e) { logError("Invalid commit " + idstr + " on patch set " + ps.getId(), e); commits.put(changeId, CodeReviewCommit.revisionGone(ctl)); continue; } // TODO(dborowitz): Consider putting ChangeData in CodeReviewCommit. commit.setControl(ctl); commit.setPatchsetId(ps.getId()); commits.put(changeId, commit); MergeValidators mergeValidators = mergeValidatorsFactory.create(); try { mergeValidators.validatePreMerge(repo, commit, destProject, destBranch, ps.getId(), caller); } catch (MergeValidationException mve) { logDebug("Revision {} of patch set {} failed validation: {}", idstr, ps.getId(), mve.getStatus()); commit.setStatusCode(mve.getStatus()); continue; } SubmitType submitType; submitType = getSubmitType(commit.getControl(), ps); if (submitType == null) { logError("No submit type for revision " + idstr + " of patch set " + ps.getId()); commit.setStatusCode(CommitMergeStatus.NO_SUBMIT_TYPE); continue; } commit.add(canMergeFlag); toSubmit.put(submitType, cd); } logDebug("Submitting on this run: {}", toSubmit); return toSubmit;
4908 1242677748GerritCodeReview/gerritDavid Pursehouse84bc19bc3d537601a7df5be670d977b113821344 TODO Technically the proper way to do this test is to use a RevWalk on \"$id --not --all\" and test for an empty set. But that is way slower than looking for a ref directly pointing at the desired tip. We should always have a ref available. TODO this is actually an error, the branch is gone but we want to merge the issue. We can't safely do that if the tip is not reachable. TODO Technically the proper way to do this test is to use a RevWalk on \"$id --not --all\" and test for an empty set. But that is way slower than looking for a ref directly pointing at the desired tip. We should always have a ref available. TODO this is actually an error, the branch is gone but we want to merge the issue. We can't safely do that if the tip is not reachable. CLASS_OR_METHOD_CHANGED ChangeData> validateChangeList(Collection, IdentifiedUser) private ListMultimap validateChangeList(Collection submitted, IdentifiedUser caller) throws IntegrationException logDebug("Validating {} changes", submitted.size()); ListMultimap toSubmit = ArrayListMultimap.create(); Map allRefs; try { allRefs = repo.getRefDatabase().getRefs(ALL); } catch (IOException e) { throw new IntegrationException(e.getMessage(), e); } Set tips = new HashSet<>(); for (Ref r : allRefs.values()) { tips.add(r.getObjectId()); } for (ChangeData cd : submitted) { ChangeControl ctl; Change chg; try { ctl = cd.changeControl(); // Reload change in case index was stale. chg = cd.reloadChange(); } catch (OrmException e) { throw new IntegrationException("Failed to validate changes", e); } Change.Id changeId = cd.getId(); if (chg.getStatus() != Change.Status.NEW) { logDebug("Change {} is not new: {}", changeId, chg.getStatus()); continue; } if (chg.currentPatchSetId() == null) { logError("Missing current patch set on change " + changeId); commits.put(changeId, CodeReviewCommit.noPatchSet(ctl)); continue; } PatchSet ps; Branch.NameKey destBranch = chg.getDest(); try { ps = cd.currentPatchSet(); } catch (OrmException e) { throw new IntegrationException("Cannot query the database", e); } if (ps == null || ps.getRevision() == null || ps.getRevision().get() == null) { logError("Missing patch set or revision on change " + changeId); commits.put(changeId, CodeReviewCommit.noPatchSet(ctl)); continue; } String idstr = ps.getRevision().get(); ObjectId id; try { id = ObjectId.fromString(idstr); } catch (IllegalArgumentException iae) { logError("Invalid revision on patch set " + ps.getId()); commits.put(changeId, CodeReviewCommit.noPatchSet(ctl)); continue; } if (!tips.contains(id)) { // TODO Technically the proper way to do this test is to use a // RevWalk on "$id --not --all" and test for an empty set. But // that is way slower than looking for a ref directly pointing // at the desired tip. We should always have a ref available. // // TODO this is actually an error, the branch is gone but we // want to merge the issue. We can't safely do that if the // tip is not reachable. // logError("Revision " + idstr + " of patch set " + ps.getId() + " is not contained in any ref"); commits.put(changeId, CodeReviewCommit.revisionGone(ctl)); continue; } CodeReviewCommit commit; try { commit = rw.parseCommit(id); } catch (IOException e) { logError("Invalid commit " + idstr + " on patch set " + ps.getId(), e); commits.put(changeId, CodeReviewCommit.revisionGone(ctl)); continue; } // TODO(dborowitz): Consider putting ChangeData in CodeReviewCommit. commit.setControl(ctl); commit.setPatchsetId(ps.getId()); commits.put(changeId, commit); MergeValidators mergeValidators = mergeValidatorsFactory.create(); try { mergeValidators.validatePreMerge(repo, commit, destProject, destBranch, ps.getId(), caller); } catch (MergeValidationException mve) { logDebug("Revision {} of patch set {} failed validation: {}", idstr, ps.getId(), mve.getStatus()); commit.setStatusCode(mve.getStatus()); continue; } SubmitType submitType; submitType = getSubmitType(commit.getControl(), ps); if (submitType == null) { logError("No submit type for revision " + idstr + " of patch set " + ps.getId()); commit.setStatusCode(CommitMergeStatus.NO_SUBMIT_TYPE); continue; } commit.add(canMergeFlag); toSubmit.put(submitType, cd); } logDebug("Submitting on this run: {}", toSubmit); return toSubmit;
4904 1242678117GerritCodeReview/gerritDave Borowitzf2839d7c16af534e208e2d2d10833d2e036ff62b None TODO(dborowitz): Can't use repo from ctx due to canMergeFlag. SATD_ADDED updateRepo(RepoContext) public void updateRepo(RepoContext ctx) throws IntegrationException, IOException // There are multiple parents, so this is a merge commit. We don't want // to rebase the merge as clients can't easily rebase their history with // that merge present and replaced by an equivalent merge with a different // first parent. So instead behave as though MERGE_IF_NECESSARY was // configured. if (args.rw.isMergedInto(mergeTip.getCurrentTip(), toMerge)) { mergeTip.moveTipTo(toMerge, toMerge); acceptMergeTip(mergeTip); } else { PersonIdent myIdent = args.serverIdent.get(); // TODO(dborowitz): Can't use repo from ctx due to canMergeFlag. CodeReviewCommit newTip = args.mergeUtil.mergeOneCommit(myIdent, myIdent, args.repo, args.rw, args.inserter, args.canMergeFlag, args.destBranch, mergeTip.getCurrentTip(), toMerge); mergeTip.moveTipTo(newTip, toMerge); } args.mergeUtil.markCleanMerges(args.rw, args.canMergeFlag, mergeTip.getCurrentTip(), args.alreadyAccepted); acceptMergeTip(mergeTip);
4903 1242678116GerritCodeReview/gerritDave Borowitzf2839d7c16af534e208e2d2d10833d2e036ff62b None TODO(dborowitz): Make RevWalk available via BatchUpdate. SATD_ADDED run(CodeReviewCommit, Collection) public MergeTip run(final CodeReviewCommit branchTip, final Collection toMerge) throws IntegrationException MergeTip mergeTip = new MergeTip(branchTip, toMerge); List sorted = sort(toMerge); boolean first = true; try (BatchUpdate u = args.newBatchUpdate(TimeUtil.nowTs())) { while (!sorted.isEmpty()) { CodeReviewCommit n = sorted.remove(0); Change.Id cid = n.change().getId(); if (first && branchTip == null) { u.addOp(cid, new RebaseUnbornRootOp(mergeTip, n)); } else if (n.getParentCount() == 0) { u.addOp(cid, new RebaseRootOp(n)); } else if (n.getParentCount() == 1) { u.addOp(cid, new RebaseOneOp(mergeTip, n)); } else { u.addOp(cid, new RebaseMultipleParentsOp(mergeTip, n)); } first = false; } u.execute(); } catch (UpdateException e) { if (e.getCause() instanceof IntegrationException) { throw new IntegrationException(e.getCause().getMessage(), e); } throw new IntegrationException("Cannot rebase onto " + args.destBranch); } catch (RestApiException e) { throw new IntegrationException("Cannot rebase onto " + args.destBranch); } return mergeTip;
4902 1242678032GerritCodeReview/gerritDavid Pursehouse4b1e2a460e30e9682f29a22d1668e80a0620d2ff TODO(dborowitz): Consider putting ChangeData in CodeReviewCommit. TODO(dborowitz): Consider putting ChangeData in CodeReviewCommit. CLASS_OR_METHOD_CHANGED validateChangeList(OpenRepo, Collection) private BranchBatch validateChangeList(OpenRepo or, Collection submitted) throws IntegrationException logDebug("Validating {} changes", submitted.size()); List toSubmit = new ArrayList<>(submitted.size()); Multimap revisions = getRevisions(or, submitted); SubmitType submitType = null; ChangeData choseSubmitTypeFrom = null; for (ChangeData cd : submitted) { ChangeControl ctl; Change chg; try { ctl = cd.changeControl(); chg = cd.change(); } catch (OrmException e) { throw new IntegrationException("Failed to validate changes", e); } Change.Id changeId = cd.getId(); if (chg.getStatus() != Change.Status.NEW) { logDebug("Change {} is not new: {}", changeId, chg.getStatus()); continue; } if (chg.currentPatchSetId() == null) { logError("Missing current patch set on change " + changeId); commits.put(changeId, CodeReviewCommit.noPatchSet(ctl)); continue; } PatchSet ps; Branch.NameKey destBranch = chg.getDest(); try { ps = cd.currentPatchSet(); } catch (OrmException e) { throw new IntegrationException("Cannot query the database", e); } if (ps == null || ps.getRevision() == null || ps.getRevision().get() == null) { logError("Missing patch set or revision on change " + changeId); commits.put(changeId, CodeReviewCommit.noPatchSet(ctl)); continue; } String idstr = ps.getRevision().get(); ObjectId id; try { id = ObjectId.fromString(idstr); } catch (IllegalArgumentException iae) { logError("Invalid revision on patch set " + ps.getId()); commits.put(changeId, CodeReviewCommit.noPatchSet(ctl)); continue; } if (!revisions.containsEntry(id, ps.getId())) { // TODO this is actually an error, the branch is gone but we // want to merge the issue. We can't safely do that if the // tip is not reachable. // logError("Revision " + idstr + " of patch set " + ps.getId() + " does not match " + ps.getId().toRefName()); commits.put(changeId, CodeReviewCommit.revisionGone(ctl)); continue; } CodeReviewCommit commit; try { commit = or.rw.parseCommit(id); } catch (IOException e) { logError("Invalid commit " + idstr + " on patch set " + ps.getId(), e); commits.put(changeId, CodeReviewCommit.revisionGone(ctl)); continue; } // TODO(dborowitz): Consider putting ChangeData in CodeReviewCommit. commit.setControl(ctl); commit.setPatchsetId(ps.getId()); commits.put(changeId, commit); MergeValidators mergeValidators = mergeValidatorsFactory.create(); try { mergeValidators.validatePreMerge(or.repo, commit, destProject, destBranch, ps.getId()); } catch (MergeValidationException mve) { logDebug("Revision {} of patch set {} failed validation: {}", idstr, ps.getId(), mve.getStatus()); commit.setStatusCode(mve.getStatus()); continue; } SubmitType st = getSubmitType(commit.getControl(), ps); if (st == null) { logError("No submit type for revision " + idstr + " of patch set " + ps.getId()); throw new IntegrationException("No submit type for change " + cd.getId()); } if (submitType == null) { submitType = st; choseSubmitTypeFrom = cd; } else if (st != submitType) { throw new IntegrationException(String.format("Change %s has submit type %s, but previously chose submit type %s " + "from change %s in the same batch", cd.getId(), st, submitType, choseSubmitTypeFrom.getId())); } commit.add(or.canMergeFlag); toSubmit.add(cd); } logDebug("Submitting on this run: {}", toSubmit); return new AutoValue_MergeOp_BranchBatch(submitType, toSubmit);
4901 1242677748GerritCodeReview/gerritDavid Pursehouse4b1e2a460e30e9682f29a22d1668e80a0620d2ff TODO this is actually an error, the branch is gone but we want to merge the issue. We can't safely do that if the tip is not reachable. TODO this is actually an error, the branch is gone but we want to merge the issue. We can't safely do that if the tip is not reachable. CLASS_OR_METHOD_CHANGED validateChangeList(OpenRepo, Collection) private BranchBatch validateChangeList(OpenRepo or, Collection submitted) throws IntegrationException logDebug("Validating {} changes", submitted.size()); List toSubmit = new ArrayList<>(submitted.size()); Multimap revisions = getRevisions(or, submitted); SubmitType submitType = null; ChangeData choseSubmitTypeFrom = null; for (ChangeData cd : submitted) { ChangeControl ctl; Change chg; try { ctl = cd.changeControl(); chg = cd.change(); } catch (OrmException e) { throw new IntegrationException("Failed to validate changes", e); } Change.Id changeId = cd.getId(); if (chg.getStatus() != Change.Status.NEW) { logDebug("Change {} is not new: {}", changeId, chg.getStatus()); continue; } if (chg.currentPatchSetId() == null) { logError("Missing current patch set on change " + changeId); commits.put(changeId, CodeReviewCommit.noPatchSet(ctl)); continue; } PatchSet ps; Branch.NameKey destBranch = chg.getDest(); try { ps = cd.currentPatchSet(); } catch (OrmException e) { throw new IntegrationException("Cannot query the database", e); } if (ps == null || ps.getRevision() == null || ps.getRevision().get() == null) { logError("Missing patch set or revision on change " + changeId); commits.put(changeId, CodeReviewCommit.noPatchSet(ctl)); continue; } String idstr = ps.getRevision().get(); ObjectId id; try { id = ObjectId.fromString(idstr); } catch (IllegalArgumentException iae) { logError("Invalid revision on patch set " + ps.getId()); commits.put(changeId, CodeReviewCommit.noPatchSet(ctl)); continue; } if (!revisions.containsEntry(id, ps.getId())) { // TODO this is actually an error, the branch is gone but we // want to merge the issue. We can't safely do that if the // tip is not reachable. // logError("Revision " + idstr + " of patch set " + ps.getId() + " does not match " + ps.getId().toRefName()); commits.put(changeId, CodeReviewCommit.revisionGone(ctl)); continue; } CodeReviewCommit commit; try { commit = or.rw.parseCommit(id); } catch (IOException e) { logError("Invalid commit " + idstr + " on patch set " + ps.getId(), e); commits.put(changeId, CodeReviewCommit.revisionGone(ctl)); continue; } // TODO(dborowitz): Consider putting ChangeData in CodeReviewCommit. commit.setControl(ctl); commit.setPatchsetId(ps.getId()); commits.put(changeId, commit); MergeValidators mergeValidators = mergeValidatorsFactory.create(); try { mergeValidators.validatePreMerge(or.repo, commit, destProject, destBranch, ps.getId()); } catch (MergeValidationException mve) { logDebug("Revision {} of patch set {} failed validation: {}", idstr, ps.getId(), mve.getStatus()); commit.setStatusCode(mve.getStatus()); continue; } SubmitType st = getSubmitType(commit.getControl(), ps); if (st == null) { logError("No submit type for revision " + idstr + " of patch set " + ps.getId()); throw new IntegrationException("No submit type for change " + cd.getId()); } if (submitType == null) { submitType = st; choseSubmitTypeFrom = cd; } else if (st != submitType) { throw new IntegrationException(String.format("Change %s has submit type %s, but previously chose submit type %s " + "from change %s in the same batch", cd.getId(), st, submitType, choseSubmitTypeFrom.getId())); } commit.add(or.canMergeFlag); toSubmit.add(cd); } logDebug("Submitting on this run: {}", toSubmit); return new AutoValue_MergeOp_BranchBatch(submitType, toSubmit);
4900 1242677748GerritCodeReview/gerritAndrew Bonventred6503968423b359569b17cf19baa309e42f61225 TODO Technically the proper way to do this test is to use a RevWalk on \"$id --not --all\" and test for an empty set. But that is way slower than looking for a ref directly pointing at the desired tip. We should always have a ref available. TODO this is actually an error, the branch is gone but we want to merge the issue. We can't safely do that if the tip is not reachable. TODO this is actually an error, the branch is gone but we want to merge the issue. We can't safely do that if the tip is not reachable. SATD_CHANGED validateChangeList(Collection) private BranchBatch validateChangeList(Collection submitted) throws IntegrationException logDebug("Validating {} changes", submitted.size()); List toSubmit = new ArrayList<>(submitted.size()); Multimap revisions = getRevisions(submitted); SubmitType submitType = null; ChangeData choseSubmitTypeFrom = null; for (ChangeData cd : submitted) { ChangeControl ctl; Change chg; try { ctl = cd.changeControl(); chg = cd.change(); } catch (OrmException e) { throw new IntegrationException("Failed to validate changes", e); } Change.Id changeId = cd.getId(); if (chg.getStatus() != Change.Status.NEW) { logDebug("Change {} is not new: {}", changeId, chg.getStatus()); continue; } if (chg.currentPatchSetId() == null) { logError("Missing current patch set on change " + changeId); commits.put(changeId, CodeReviewCommit.noPatchSet(ctl)); continue; } PatchSet ps; Branch.NameKey destBranch = chg.getDest(); try { ps = cd.currentPatchSet(); } catch (OrmException e) { throw new IntegrationException("Cannot query the database", e); } if (ps == null || ps.getRevision() == null || ps.getRevision().get() == null) { logError("Missing patch set or revision on change " + changeId); commits.put(changeId, CodeReviewCommit.noPatchSet(ctl)); continue; } String idstr = ps.getRevision().get(); ObjectId id; try { id = ObjectId.fromString(idstr); } catch (IllegalArgumentException iae) { logError("Invalid revision on patch set " + ps.getId()); commits.put(changeId, CodeReviewCommit.noPatchSet(ctl)); continue; } if (!revisions.containsEntry(id, ps.getId())) { // TODO this is actually an error, the branch is gone but we // want to merge the issue. We can't safely do that if the // tip is not reachable. // logError("Revision " + idstr + " of patch set " + ps.getId() + " does not match " + ps.getId().toRefName()); commits.put(changeId, CodeReviewCommit.revisionGone(ctl)); continue; } CodeReviewCommit commit; try { commit = rw.parseCommit(id); } catch (IOException e) { logError("Invalid commit " + idstr + " on patch set " + ps.getId(), e); commits.put(changeId, CodeReviewCommit.revisionGone(ctl)); continue; } // TODO(dborowitz): Consider putting ChangeData in CodeReviewCommit. commit.setControl(ctl); commit.setPatchsetId(ps.getId()); commits.put(changeId, commit); MergeValidators mergeValidators = mergeValidatorsFactory.create(); try { mergeValidators.validatePreMerge(repo, commit, destProject, destBranch, ps.getId()); } catch (MergeValidationException mve) { logDebug("Revision {} of patch set {} failed validation: {}", idstr, ps.getId(), mve.getStatus()); commit.setStatusCode(mve.getStatus()); continue; } SubmitType st = getSubmitType(commit.getControl(), ps); if (st == null) { logError("No submit type for revision " + idstr + " of patch set " + ps.getId()); throw new IntegrationException("No submit type for change " + cd.getId()); } if (submitType == null) { submitType = st; choseSubmitTypeFrom = cd; } else if (st != submitType) { throw new IntegrationException(String.format("Change %s has submit type %s, but previously chose submit type %s " + "from change %s in the same batch", cd.getId(), st, submitType, choseSubmitTypeFrom.getId())); } commit.add(canMergeFlag); toSubmit.add(cd); } logDebug("Submitting on this run: {}", toSubmit); return new AutoValue_MergeOp_BranchBatch(submitType, toSubmit);
4899 1242678032GerritCodeReview/gerritDave Borowitz0761fd5ce4a7a34fa5b776c0bcfdc3b6e18d8697 TODO(dborowitz): Consider putting ChangeData in CodeReviewCommit. TODO(dborowitz): Consider putting ChangeData in CodeReviewCommit. CLASS_OR_METHOD_CHANGED validateChangeList(Collection) private BranchBatch validateChangeList(Collection submitted) throws IntegrationException logDebug("Validating {} changes", submitted.size()); List toSubmit = new ArrayList<>(submitted.size()); Map allRefs; try { allRefs = repo.getRefDatabase().getRefs(ALL); } catch (IOException e) { throw new IntegrationException(e.getMessage(), e); } Set tips = new HashSet<>(); for (Ref r : allRefs.values()) { tips.add(r.getObjectId()); } SubmitType submitType = null; ChangeData choseSubmitTypeFrom = null; for (ChangeData cd : submitted) { ChangeControl ctl; Change chg; try { ctl = cd.changeControl(); // Reload change in case index was stale. chg = cd.reloadChange(); } catch (OrmException e) { throw new IntegrationException("Failed to validate changes", e); } Change.Id changeId = cd.getId(); if (chg.getStatus() != Change.Status.NEW) { logDebug("Change {} is not new: {}", changeId, chg.getStatus()); continue; } if (chg.currentPatchSetId() == null) { logError("Missing current patch set on change " + changeId); commits.put(changeId, CodeReviewCommit.noPatchSet(ctl)); continue; } PatchSet ps; Branch.NameKey destBranch = chg.getDest(); try { ps = cd.currentPatchSet(); } catch (OrmException e) { throw new IntegrationException("Cannot query the database", e); } if (ps == null || ps.getRevision() == null || ps.getRevision().get() == null) { logError("Missing patch set or revision on change " + changeId); commits.put(changeId, CodeReviewCommit.noPatchSet(ctl)); continue; } String idstr = ps.getRevision().get(); ObjectId id; try { id = ObjectId.fromString(idstr); } catch (IllegalArgumentException iae) { logError("Invalid revision on patch set " + ps.getId()); commits.put(changeId, CodeReviewCommit.noPatchSet(ctl)); continue; } if (!tips.contains(id)) { // TODO Technically the proper way to do this test is to use a // RevWalk on "$id --not --all" and test for an empty set. But // that is way slower than looking for a ref directly pointing // at the desired tip. We should always have a ref available. // // TODO this is actually an error, the branch is gone but we // want to merge the issue. We can't safely do that if the // tip is not reachable. // logError("Revision " + idstr + " of patch set " + ps.getId() + " is not contained in any ref"); commits.put(changeId, CodeReviewCommit.revisionGone(ctl)); continue; } CodeReviewCommit commit; try { commit = rw.parseCommit(id); } catch (IOException e) { logError("Invalid commit " + idstr + " on patch set " + ps.getId(), e); commits.put(changeId, CodeReviewCommit.revisionGone(ctl)); continue; } // TODO(dborowitz): Consider putting ChangeData in CodeReviewCommit. commit.setControl(ctl); commit.setPatchsetId(ps.getId()); commits.put(changeId, commit); MergeValidators mergeValidators = mergeValidatorsFactory.create(); try { mergeValidators.validatePreMerge(repo, commit, destProject, destBranch, ps.getId()); } catch (MergeValidationException mve) { logDebug("Revision {} of patch set {} failed validation: {}", idstr, ps.getId(), mve.getStatus()); commit.setStatusCode(mve.getStatus()); continue; } SubmitType st = getSubmitType(commit.getControl(), ps); if (st == null) { logError("No submit type for revision " + idstr + " of patch set " + ps.getId()); throw new IntegrationException("No submit type for change " + cd.getId()); } if (submitType == null) { submitType = st; choseSubmitTypeFrom = cd; } else if (st != submitType) { throw new IntegrationException(String.format("Change %s has submit type %s, but previously chose submit type %s " + "from change %s in the same batch", cd.getId(), st, submitType, choseSubmitTypeFrom.getId())); } commit.add(canMergeFlag); toSubmit.add(cd); } logDebug("Submitting on this run: {}", toSubmit); return new AutoValue_MergeOp_BranchBatch(submitType, toSubmit);
4898 1242677748GerritCodeReview/gerritDave Borowitz0761fd5ce4a7a34fa5b776c0bcfdc3b6e18d8697 TODO Technically the proper way to do this test is to use a RevWalk on \"$id --not --all\" and test for an empty set. But that is way slower than looking for a ref directly pointing at the desired tip. We should always have a ref available. TODO this is actually an error, the branch is gone but we want to merge the issue. We can't safely do that if the tip is not reachable. TODO Technically the proper way to do this test is to use a RevWalk on \"$id --not --all\" and test for an empty set. But that is way slower than looking for a ref directly pointing at the desired tip. We should always have a ref available. TODO this is actually an error, the branch is gone but we want to merge the issue. We can't safely do that if the tip is not reachable. CLASS_OR_METHOD_CHANGED validateChangeList(Collection) private BranchBatch validateChangeList(Collection submitted) throws IntegrationException logDebug("Validating {} changes", submitted.size()); List toSubmit = new ArrayList<>(submitted.size()); Map allRefs; try { allRefs = repo.getRefDatabase().getRefs(ALL); } catch (IOException e) { throw new IntegrationException(e.getMessage(), e); } Set tips = new HashSet<>(); for (Ref r : allRefs.values()) { tips.add(r.getObjectId()); } SubmitType submitType = null; ChangeData choseSubmitTypeFrom = null; for (ChangeData cd : submitted) { ChangeControl ctl; Change chg; try { ctl = cd.changeControl(); // Reload change in case index was stale. chg = cd.reloadChange(); } catch (OrmException e) { throw new IntegrationException("Failed to validate changes", e); } Change.Id changeId = cd.getId(); if (chg.getStatus() != Change.Status.NEW) { logDebug("Change {} is not new: {}", changeId, chg.getStatus()); continue; } if (chg.currentPatchSetId() == null) { logError("Missing current patch set on change " + changeId); commits.put(changeId, CodeReviewCommit.noPatchSet(ctl)); continue; } PatchSet ps; Branch.NameKey destBranch = chg.getDest(); try { ps = cd.currentPatchSet(); } catch (OrmException e) { throw new IntegrationException("Cannot query the database", e); } if (ps == null || ps.getRevision() == null || ps.getRevision().get() == null) { logError("Missing patch set or revision on change " + changeId); commits.put(changeId, CodeReviewCommit.noPatchSet(ctl)); continue; } String idstr = ps.getRevision().get(); ObjectId id; try { id = ObjectId.fromString(idstr); } catch (IllegalArgumentException iae) { logError("Invalid revision on patch set " + ps.getId()); commits.put(changeId, CodeReviewCommit.noPatchSet(ctl)); continue; } if (!tips.contains(id)) { // TODO Technically the proper way to do this test is to use a // RevWalk on "$id --not --all" and test for an empty set. But // that is way slower than looking for a ref directly pointing // at the desired tip. We should always have a ref available. // // TODO this is actually an error, the branch is gone but we // want to merge the issue. We can't safely do that if the // tip is not reachable. // logError("Revision " + idstr + " of patch set " + ps.getId() + " is not contained in any ref"); commits.put(changeId, CodeReviewCommit.revisionGone(ctl)); continue; } CodeReviewCommit commit; try { commit = rw.parseCommit(id); } catch (IOException e) { logError("Invalid commit " + idstr + " on patch set " + ps.getId(), e); commits.put(changeId, CodeReviewCommit.revisionGone(ctl)); continue; } // TODO(dborowitz): Consider putting ChangeData in CodeReviewCommit. commit.setControl(ctl); commit.setPatchsetId(ps.getId()); commits.put(changeId, commit); MergeValidators mergeValidators = mergeValidatorsFactory.create(); try { mergeValidators.validatePreMerge(repo, commit, destProject, destBranch, ps.getId()); } catch (MergeValidationException mve) { logDebug("Revision {} of patch set {} failed validation: {}", idstr, ps.getId(), mve.getStatus()); commit.setStatusCode(mve.getStatus()); continue; } SubmitType st = getSubmitType(commit.getControl(), ps); if (st == null) { logError("No submit type for revision " + idstr + " of patch set " + ps.getId()); throw new IntegrationException("No submit type for change " + cd.getId()); } if (submitType == null) { submitType = st; choseSubmitTypeFrom = cd; } else if (st != submitType) { throw new IntegrationException(String.format("Change %s has submit type %s, but previously chose submit type %s " + "from change %s in the same batch", cd.getId(), st, submitType, choseSubmitTypeFrom.getId())); } commit.add(canMergeFlag); toSubmit.add(cd); } logDebug("Submitting on this run: {}", toSubmit); return new AutoValue_MergeOp_BranchBatch(submitType, toSubmit);
4897 1242678115GerritCodeReview/gerritDavid Pursehousee6ac4ad532f01d5efeaaa6033dfb1e167d2dc0c8 None This is (for now) a fatal error. There are two records for what might be the same user. The admin would have to link the accounts manually. SATD_ADDED authenticateAndRedirect(HttpServletRequest, HttpServletResponse) private void authenticateAndRedirect(HttpServletRequest req, HttpServletResponse rsp) throws IOException com.google.gerrit.server.account.AuthRequest areq = new com.google.gerrit.server.account.AuthRequest(user.getExternalId()); AuthResult arsp = null; try { String claimedIdentifier = user.getClaimedIdentity(); Account.Id actualId = accountManager.lookup(user.getExternalId()); Account.Id claimedId = null; // We try to retrieve claimed identity. // For some reason, for example staging instance // it may deviate from the really old OpenID identity. // What we want to avoid in any event is to create new // account instead of linking to the existing one. // That why we query it here, not to lose linking mode. if (!Strings.isNullOrEmpty(claimedIdentifier)) { claimedId = accountManager.lookup(claimedIdentifier); if (claimedId == null) { log.debug("Claimed identity is unknown"); } } // Use case 1: claimed identity was provided during handshake phase // and user account exists for this identity if (claimedId != null) { log.debug("Claimed identity is set and is known"); if (actualId != null) { if (claimedId.equals(actualId)) { // Both link to the same account, that's what we expected. } else { // This is (for now) a fatal error. There are two records // for what might be the same user. The admin would have to // link the accounts manually. log.error("OAuth accounts disagree over user identity:\n" + " Claimed ID: " + claimedId + " is " + claimedIdentifier + "\n" + " Delgate ID: " + actualId + " is " + user.getExternalId()); rsp.sendError(HttpServletResponse.SC_FORBIDDEN); return; } } else { // Claimed account already exists: link to it. // try { accountManager.link(claimedId, areq); } catch (OrmException e) { log.error("Cannot link: " + user.getExternalId() + " to user identity:\n" + " Claimed ID: " + claimedId + " is " + claimedIdentifier); rsp.sendError(HttpServletResponse.SC_FORBIDDEN); return; } } } else if (linkMode) { // Use case 2: link mode activated from the UI try { accountManager.link(identifiedUser.get().getAccountId(), areq); } catch (OrmException e) { log.error("Cannot link: " + user.getExternalId() + " to user identity: " + identifiedUser.get().getAccountId()); rsp.sendError(HttpServletResponse.SC_FORBIDDEN); return; } finally { linkMode = false; } } areq.setUserName(user.getUserName()); areq.setEmailAddress(user.getEmailAddress()); areq.setDisplayName(user.getDisplayName()); arsp = accountManager.authenticate(areq); } catch (AccountException e) { log.error("Unable to authenticate user \"" + user + "\"", e); rsp.sendError(HttpServletResponse.SC_FORBIDDEN); return; } webSession.get().login(arsp, true); StringBuilder rdr = new StringBuilder(urlProvider.get(req)); rdr.append(Url.decode(redirectToken)); rsp.sendRedirect(rdr.toString());
4896 1242678114GerritCodeReview/gerritDavid Pursehousee6ac4ad532f01d5efeaaa6033dfb1e167d2dc0c8 None We try to retrieve claimed identity. For some reason, for example staging instance it may deviate from the really old OpenID identity. What we want to avoid in any event is to create new account instead of linking to the existing one. That why we query it here, not to lose linking mode. SATD_ADDED authenticateAndRedirect(HttpServletRequest, HttpServletResponse) private void authenticateAndRedirect(HttpServletRequest req, HttpServletResponse rsp) throws IOException com.google.gerrit.server.account.AuthRequest areq = new com.google.gerrit.server.account.AuthRequest(user.getExternalId()); AuthResult arsp = null; try { String claimedIdentifier = user.getClaimedIdentity(); Account.Id actualId = accountManager.lookup(user.getExternalId()); Account.Id claimedId = null; // We try to retrieve claimed identity. // For some reason, for example staging instance // it may deviate from the really old OpenID identity. // What we want to avoid in any event is to create new // account instead of linking to the existing one. // That why we query it here, not to lose linking mode. if (!Strings.isNullOrEmpty(claimedIdentifier)) { claimedId = accountManager.lookup(claimedIdentifier); if (claimedId == null) { log.debug("Claimed identity is unknown"); } } // Use case 1: claimed identity was provided during handshake phase // and user account exists for this identity if (claimedId != null) { log.debug("Claimed identity is set and is known"); if (actualId != null) { if (claimedId.equals(actualId)) { // Both link to the same account, that's what we expected. } else { // This is (for now) a fatal error. There are two records // for what might be the same user. The admin would have to // link the accounts manually. log.error("OAuth accounts disagree over user identity:\n" + " Claimed ID: " + claimedId + " is " + claimedIdentifier + "\n" + " Delgate ID: " + actualId + " is " + user.getExternalId()); rsp.sendError(HttpServletResponse.SC_FORBIDDEN); return; } } else { // Claimed account already exists: link to it. // try { accountManager.link(claimedId, areq); } catch (OrmException e) { log.error("Cannot link: " + user.getExternalId() + " to user identity:\n" + " Claimed ID: " + claimedId + " is " + claimedIdentifier); rsp.sendError(HttpServletResponse.SC_FORBIDDEN); return; } } } else if (linkMode) { // Use case 2: link mode activated from the UI try { accountManager.link(identifiedUser.get().getAccountId(), areq); } catch (OrmException e) { log.error("Cannot link: " + user.getExternalId() + " to user identity: " + identifiedUser.get().getAccountId()); rsp.sendError(HttpServletResponse.SC_FORBIDDEN); return; } finally { linkMode = false; } } areq.setUserName(user.getUserName()); areq.setEmailAddress(user.getEmailAddress()); areq.setDisplayName(user.getDisplayName()); arsp = accountManager.authenticate(areq); } catch (AccountException e) { log.error("Unable to authenticate user \"" + user + "\"", e); rsp.sendError(HttpServletResponse.SC_FORBIDDEN); return; } webSession.get().login(arsp, true); StringBuilder rdr = new StringBuilder(urlProvider.get(req)); rdr.append(Url.decode(redirectToken)); rsp.sendRedirect(rdr.toString());
4895 1242678113GerritCodeReview/gerritMichael Ochmann6c05d5b5412247492f892a42d2db0d51619b0d22 None no providerId in the cookie value => assume default provider note: a leading/trailing at sign is considered to belong to the access token rather than being a separator SATD_ADDED init(FilterConfig) public void init(FilterConfig config) throws ServletException if (Strings.isNullOrEmpty(gitOAuthProvider)) { pickOnlyProvider(); } else { pickConfiguredProvider(); }
4892 1242678112GerritCodeReview/gerritEdwin Kempinbc8006f7e84746b9d56810744dc91af7f3bb77b3 None TODO(dborowitz): ResourceConflictException. SATD_ADDED deleteDraftChange(Change) public void deleteDraftChange(Change change) throws NoSuchChangeException, OrmException, IOException ReviewDb db = this.db.get(); Change.Id changeId = change.getId(); if (change.getStatus() != Change.Status.DRAFT) { // TODO(dborowitz): ResourceConflictException. throw new NoSuchChangeException(changeId); } List patchSets = db.patchSets().byChange(changeId).toList(); for (PatchSet ps : patchSets) { if (!ps.isDraft()) { // TODO(dborowitz): ResourceConflictException. throw new NoSuchChangeException(changeId); } db.accountPatchReviews().delete(db.accountPatchReviews().byPatchSet(ps.getId())); } // No need to delete from notedb; draft patch sets will be filtered out. db.patchComments().delete(db.patchComments().byChange(changeId)); db.patchSetApprovals().delete(db.patchSetApprovals().byChange(changeId)); db.patchSets().delete(patchSets); db.changeMessages().delete(db.changeMessages().byChange(changeId)); starredChangesUtil.unstarAll(changeId); db.changes().delete(Collections.singleton(change)); // Delete all refs at once. try (Repository repo = gitManager.openRepository(change.getProject()); RevWalk rw = new RevWalk(repo)) { String prefix = new PatchSet.Id(changeId, 1).toRefName(); prefix = prefix.substring(0, prefix.length() - 1); BatchRefUpdate ru = repo.getRefDatabase().newBatchUpdate(); for (Ref ref : repo.getRefDatabase().getRefs(prefix).values()) { ru.addCommand(new ReceiveCommand(ref.getObjectId(), ObjectId.zeroId(), ref.getName())); } ru.execute(rw, NullProgressMonitor.INSTANCE); for (ReceiveCommand cmd : ru.getCommands()) { if (cmd.getResult() != ReceiveCommand.Result.OK) { throw new IOException("failed: " + cmd + ": " + cmd.getResult()); } } }
4891 1242678111GerritCodeReview/gerritEdwin Kempinbc8006f7e84746b9d56810744dc91af7f3bb77b3 None TODO(dborowitz): Delete after migrating DeleteDraftChange to BatchUpdate. SATD_ADDED deleteDraftChangeInNewTransaction(Change) public void deleteDraftChangeInNewTransaction(Change change) throws NoSuchChangeException, OrmException, IOException db.get().changes().beginTransaction(change.getId()); try { deleteDraftChange(change); db.get().commit(); indexer.delete(change.getId()); } finally { db.get().rollback(); }
4889 1242678110GerritCodeReview/gerritMichael Zhou713551b09573d6f67916d562908f5618d6adee70 None limit the number of query terms otherwise Gerrit will barf SATD_ADDED label(String) public Predicate label(String name) throws QueryParseException, OrmException Set accounts = null; AccountGroup.UUID group = null; // Parse for: // label:CodeReview=1,user=jsmith or // label:CodeReview=1,jsmith or // label:CodeReview=1,group=android_approvers or // label:CodeReview=1,android_approvers // user/groups without a label will first attempt to match user String[] splitReviewer = name.split(",", 2); // remove all but the vote piece, e.g.'CodeReview=1' name = splitReviewer[0]; if (splitReviewer.length == 2) { // process the user/group piece PredicateArgs lblArgs = new PredicateArgs(splitReviewer[1]); for (Map.Entry pair : lblArgs.keyValue.entrySet()) { if (pair.getKey().equalsIgnoreCase(ARG_ID_USER)) { accounts = parseAccount(pair.getValue()); } else if (pair.getKey().equalsIgnoreCase(ARG_ID_GROUP)) { group = parseGroup(pair.getValue()).getUUID(); } else { throw new QueryParseException("Invalid argument identifier '" + pair.getKey() + "'"); } } for (String value : lblArgs.positional) { if (accounts != null || group != null) { throw new QueryParseException("more than one user/group specified (" + value + ")"); } try { accounts = parseAccount(value); } catch (QueryParseException qpex) { // If it doesn't match an account, see if it matches a group // (accounts get precedence) try { group = parseGroup(value).getUUID(); } catch (QueryParseException e) { throw error("Neither user nor group " + value + " found"); } } } } // expand a group predicate into multiple user predicates if (group != null) { Set allMembers = getMemberIds(group, new HashSet()); int maxTerms = args.indexConfig.maxLimit(); if (allMembers.size() > maxTerms) { // limit the number of query terms otherwise Gerrit will barf accounts = ImmutableSet.copyOf(Iterables.limit(allMembers, maxTerms)); } else { accounts = allMembers; } } return new LabelPredicate(args.projectCache, args.changeControlGenericFactory, args.userFactory, args.db, name, accounts, group);
4887 1242678109GerritCodeReview/gerritDave Borowitzc916b9e6a91864024185b820ccede36546805794 Serve static resources directly from our JAR. This way we don't need to unpack them into yet another temporary directory prior to serving to clients. This is the path we are accessed by clients within our domain. SATD_REMOVED makeContext(String, JettyEnv, Config) private ContextHandler makeContext(final String contextPath, final JettyEnv env, final Config cfg) final ServletContextHandler app = new ServletContextHandler(); // This enables the use of sessions in Jetty, feature available // for Gerrit plug-ins to enable user-level sessions. // app.setSessionHandler(new SessionHandler()); app.setErrorHandler(new HiddenErrorHandler()); // This is the path we are accessed by clients within our domain. // app.setContextPath(contextPath); // HTTP front-end filter to be used as surrogate of Apache HTTP // reverse-proxy filtering. // It is meant to be used as simpler tiny deployment of custom-made // security enforcement (Security tokens, IP-based security filtering, others) String filterClassName = cfg.getString("httpd", null, "filterClass"); if (filterClassName != null) { try { @SuppressWarnings("unchecked") Class filterClass = (Class) Class.forName(filterClassName); Filter filter = env.webInjector.getInstance(filterClass); app.addFilter(new FilterHolder(filter), "/*", EnumSet.of(DispatcherType.REQUEST, DispatcherType.ASYNC)); } catch (Throwable e) { String errorMessage = "Unable to instantiate front-end HTTP Filter " + filterClassName; log.error(errorMessage, e); throw new IllegalArgumentException(errorMessage, e); } } // Perform the same binding as our web.xml would do, but instead // of using the listener to create the injector pass the one we // already have built. // GuiceFilter filter = env.webInjector.getInstance(GuiceFilter.class); app.addFilter(new FilterHolder(filter), "/*", EnumSet.of(DispatcherType.REQUEST, DispatcherType.ASYNC)); app.addEventListener(new GuiceServletContextListener() { @Override protected Injector getInjector() { return env.webInjector; } }); // Jetty requires at least one servlet be bound before it will // bother running the filter above. Since the filter has all // of our URLs except the static resources, the only servlet // we need to bind is the default static resource servlet from // the Jetty container. // final ServletHolder ds = app.addServlet(DefaultServlet.class, "/"); ds.setInitParameter("dirAllowed", "false"); ds.setInitParameter("redirectWelcome", "false"); ds.setInitParameter("useFileMappedBuffer", "false"); ds.setInitParameter("gzip", "true"); app.setWelcomeFiles(new String[0]); return app;
4885 1242678108GerritCodeReview/gerritDavid Ostrovsky7d5a77184483bb24eb1335359d01dfbf356daa99 None TODO(davido): Remove manual merging in follow-up change SATD_ADDED merge(DiffPreferencesInfo, DiffPreferencesInfo) private DiffPreferencesInfo merge(DiffPreferencesInfo n, DiffPreferencesInfo i) if (i.context != null) { n.context = i.context; } if (i.expandAllComments != null) { n.expandAllComments = i.expandAllComments; } if (i.hideLineNumbers != null) { n.hideLineNumbers = i.hideLineNumbers; } if (i.hideTopMenu != null) { n.hideTopMenu = i.hideTopMenu; } if (i.ignoreWhitespace != null) { n.ignoreWhitespace = i.ignoreWhitespace; } if (i.intralineDifference != null) { n.intralineDifference = i.intralineDifference; } if (i.lineLength != null) { n.lineLength = i.lineLength; } if (i.manualReview != null) { n.manualReview = i.manualReview; } if (i.renderEntireFile != null) { n.renderEntireFile = i.renderEntireFile; } if (i.retainHeader != null) { n.retainHeader = i.retainHeader; } if (i.showLineEndings != null) { n.showLineEndings = i.showLineEndings; } if (i.showTabs != null) { n.showTabs = i.showTabs; } if (i.showWhitespaceErrors != null) { n.showWhitespaceErrors = i.showWhitespaceErrors; } if (i.skipDeleted != null) { n.skipDeleted = i.skipDeleted; } if (i.skipUncommented != null) { n.skipUncommented = i.skipUncommented; } if (i.syntaxHighlighting != null) { n.syntaxHighlighting = i.syntaxHighlighting; } if (i.tabSize != null) { n.tabSize = i.tabSize; } if (i.theme != null) { n.theme = i.theme; } if (i.hideEmptyPane != null) { n.hideEmptyPane = i.hideEmptyPane; } if (i.autoHideDiffTableHeader != null) { n.autoHideDiffTableHeader = i.autoHideDiffTableHeader; } return n;
4884 1242678107GerritCodeReview/gerritDave Borowitz459b191c5e2a0628915e5abe28ac7b1e14bd241c Breadth-first search with oldest children first. TODO(dborowitz): After killing PatchSetAncestors, consider DFS to keep parallel history together. Depth-first search with newest children first. SATD_REMOVED walkDescendentsImpl(ProjectControl, Set, ListMultimap, List) private static List walkDescendentsImpl(ProjectControl ctl, Set alreadyEmittedChanges, ListMultimap children, List start) throws OrmException if (start.isEmpty()) { return ImmutableList.of(); } Map maxPatchSetIds = new HashMap<>(); List allPatchSets = new ArrayList<>(); Deque pending = new ArrayDeque<>(); pending.addAll(start); while (!pending.isEmpty()) { PatchSetData psd = pending.remove(); if (!isVisible(psd, ctl)) { continue; } if (!alreadyEmittedChanges.contains(psd.id())) { // Don't emit anything for changes that were previously emitted, even // though different patch sets might show up later. However, do // continue walking through them for the purposes of finding indirect // descendants. PatchSet.Id oldMax = maxPatchSetIds.get(psd.id()); if (oldMax == null || psd.psId().get() > oldMax.get()) { maxPatchSetIds.put(psd.id(), psd.psId()); } allPatchSets.add(psd); } // Depth-first search with newest children first. for (PatchSetData child : children.get(psd)) { pending.addFirst(child); } } // If we saw the same change multiple times, prefer the latest patch set. List result = new ArrayList<>(allPatchSets.size()); for (PatchSetData psd : allPatchSets) { if (checkNotNull(maxPatchSetIds.get(psd.id())).equals(psd.psId())) { result.add(psd); } } return result;
4883 1242678107GerritCodeReview/gerritDave Borowitz17db2b06ec7198b6ba0d5617f6f803a81df89e76 Breadth-first search with oldest children first. TODO(dborowitz): After killing PatchSetAncestors, consider DFS to keep parallel history together. Breadth-first search with oldest children first. TODO(dborowitz): After killing PatchSetAncestors, consider DFS to keep parallel history together. FILE_PATH_CHANGED walkDescendentsImpl(ProjectControl, Set, ListMultimap, List) private static List walkDescendentsImpl(ProjectControl ctl, Set alreadyEmittedChanges, ListMultimap children, List start) throws OrmException if (start.isEmpty()) { return ImmutableList.of(); } Map maxPatchSetIds = new HashMap<>(); List allPatchSets = new ArrayList<>(); Deque pending = new ArrayDeque<>(); pending.addAll(start); while (!pending.isEmpty()) { PatchSetData psd = pending.remove(); if (!isVisible(psd, ctl)) { continue; } if (!alreadyEmittedChanges.contains(psd.id())) { // Don't emit anything for changes that were previously emitted, even // though different patch sets might show up later. However, do // continue walking through them for the purposes of finding indirect // descendants. PatchSet.Id oldMax = maxPatchSetIds.get(psd.id()); if (oldMax == null || psd.psId().get() > oldMax.get()) { maxPatchSetIds.put(psd.id(), psd.psId()); } allPatchSets.add(psd); } // Breadth-first search with oldest children first. // TODO(dborowitz): After killing PatchSetAncestors, consider DFS to keep // parallel history together. pending.addAll(Lists.reverse(children.get(psd))); } // If we saw the same change multiple times, prefer the latest patch set. List result = new ArrayList<>(allPatchSets.size()); for (PatchSetData psd : allPatchSets) { if (checkNotNull(maxPatchSetIds.get(psd.id())).equals(psd.psId())) { result.add(psd); } } return result;
4881 1242678062GerritCodeReview/gerritDavid Ostrovsky39db6c73e65e00812314d3f07c231402d4d3bafb TODO(dborowitz): TEST. TODO(dborowitz): TEST. CLASS_OR_METHOD_CHANGED noteDbEnabled() public static Config noteDbEnabled() return NotesMigration.allEnabledConfig();
4880 1242678107GerritCodeReview/gerritDavid Pursehouse39d1a499a74ce3586be1571e7bf9d5ac5269d5ef None Breadth-first search with oldest children first. TODO(dborowitz): After killing PatchSetAncestors, consider DFS to keep parallel history together. SATD_ADDED walkDescendentsImpl(ProjectControl, Set, ListMultimap, List) private static List walkDescendentsImpl(ProjectControl ctl, Set alreadyEmittedChanges, ListMultimap children, List start) throws OrmException if (start.isEmpty()) { return ImmutableList.of(); } Map maxPatchSetIds = new HashMap<>(); List allPatchSets = new ArrayList<>(); Deque pending = new ArrayDeque<>(); pending.addAll(start); while (!pending.isEmpty()) { PatchSetData psd = pending.remove(); if (!isVisible(psd, ctl)) { continue; } if (!alreadyEmittedChanges.contains(psd.id())) { // Don't emit anything for changes that were previously emitted, even // though different patch sets might show up later. However, do // continue walking through them for the purposes of finding indirect // descendants. PatchSet.Id oldMax = maxPatchSetIds.get(psd.id()); if (oldMax == null || psd.psId().get() > oldMax.get()) { maxPatchSetIds.put(psd.id(), psd.psId()); } allPatchSets.add(psd); } // Breadth-first search with oldest children first. // TODO(dborowitz): After killing PatchSetAncestors, consider DFS to keep // parallel history together. pending.addAll(Lists.reverse(children.get(psd))); } // If we saw the same change multiple times, prefer the latest patch set. List result = new ArrayList<>(allPatchSets.size()); for (PatchSetData psd : allPatchSets) { if (checkNotNull(maxPatchSetIds.get(psd.id())).equals(psd.psId())) { result.add(psd); } } return result;
4879 1242678106GerritCodeReview/gerritDavid Pursehouse39d1a499a74ce3586be1571e7bf9d5ac5269d5ef None 1,2 is related directly to 4,1, and the 2-3 parallel branch stays intact. SATD_ADDED getRelatedReworkThenExtendInTheMiddleOfSeries() public void getRelatedReworkThenExtendInTheMiddleOfSeries() throws Exception // 1,1---2,1---3,1 // // 1,2---2,2---3,2 // \---4,1 // Create three commits and push. RevCommit c1_1 = commitBuilder().add("a.txt", "1").message("subject: 1").create(); RevCommit c2_1 = commitBuilder().add("b.txt", "1").message("subject: 2").create(); RevCommit c3_1 = commitBuilder().add("b.txt", "1").message("subject: 3").create(); pushHead(testRepo, "refs/for/master", false); PatchSet.Id ps1_1 = getPatchSetId(c1_1); PatchSet.Id ps2_1 = getPatchSetId(c2_1); PatchSet.Id ps3_1 = getPatchSetId(c3_1); // Amend all changes change and push. testRepo.reset(c1_1); RevCommit c1_2 = amendBuilder().add("a.txt", "2").create(); RevCommit c2_2 = commitBuilder().add("b.txt", "2").message(parseBody(c2_1).getFullMessage()).create(); RevCommit c3_2 = commitBuilder().add("b.txt", "3").message(parseBody(c3_1).getFullMessage()).create(); pushHead(testRepo, "refs/for/master", false); PatchSet.Id ps1_2 = getPatchSetId(c1_2); PatchSet.Id ps2_2 = getPatchSetId(c2_2); PatchSet.Id ps3_2 = getPatchSetId(c3_2); // Add one more commit 4,1 based on 1,2. testRepo.reset(c1_2); RevCommit c4_1 = commitBuilder().add("d.txt", "4").message("subject: 4").create(); pushHead(testRepo, "refs/for/master", false); PatchSet.Id ps4_1 = getPatchSetId(c4_1); // 1,1 is related indirectly to 4,1. assertRelated(ps1_1, changeAndCommit(ps4_1, c4_1, 1), changeAndCommit(ps3_1, c3_1, 2), changeAndCommit(ps2_1, c2_1, 2), changeAndCommit(ps1_1, c1_1, 2)); // 2,1 and 3,1 don't include 4,1 since we don't walk forward after walking // backward. for (PatchSet.Id ps : ImmutableList.of(ps2_1, ps3_1)) { assertRelated(ps, changeAndCommit(ps3_1, c3_1, 2), changeAndCommit(ps2_1, c2_1, 2), changeAndCommit(ps1_1, c1_1, 2)); } // 1,2 is related directly to 4,1, and the 2-3 parallel branch stays intact. assertRelated(ps1_2, changeAndCommit(ps3_2, c3_2, 2), changeAndCommit(ps4_1, c4_1, 1), changeAndCommit(ps2_2, c2_2, 2), changeAndCommit(ps1_2, c1_2, 2)); // 4,1 is only related to 1,2, since we don't walk forward after walking // backward. assertRelated(ps4_1, changeAndCommit(ps4_1, c4_1, 1), changeAndCommit(ps1_2, c1_2, 2)); // 2,2 and 3,2 don't include 4,1 since we don't walk forward after walking // backward. for (PatchSet.Id ps : ImmutableList.of(ps2_2, ps3_2)) { assertRelated(ps, changeAndCommit(ps3_2, c3_2, 2), changeAndCommit(ps2_2, c2_2, 2), changeAndCommit(ps1_2, c1_2, 2)); }
4877 1242678105GerritCodeReview/gerritDave Borowitz5ed11440fc10cd13b225db6b59238252f8274d86 None RFC4880 states: \"If a key has been revoked because of a compromise, all signatures created by that key are suspect. However, if it was merely superseded or retired, old signatures are still valid.\" Note that GnuPG does not implement this correctly, as it does not consider the revocation reason and timestamp when checking whether a signature (data or certification) is valid. SATD_ADDED isRevocationValid(PGPSignature, RevocationReason, Date) private static boolean isRevocationValid(PGPSignature revocation, RevocationReason reason, Date now) // RFC4880 states: // "If a key has been revoked because of a compromise, all signatures // created by that key are suspect. However, if it was merely superseded or // retired, old signatures are still valid." // // Note that GnuPG does not implement this correctly, as it does not // consider the revocation reason and timestamp when checking whether a // signature (data or certification) is valid. return reason.getRevocationReason() == KEY_COMPROMISED || revocation.getCreationTime().before(now);
4875 1242677883GerritCodeReview/gerritDave Borowitz9d12abd177cea3ff708b6b762e1a5a6179a1942d If no existing label is being set to 0, hack in the caller as a reviewer by picking the first server-wide LabelType. If no existing label is being set to 0, hack in the caller as a reviewer by picking the first server-wide LabelType. CLASS_OR_METHOD_CHANGED forceCallerAsReviewer(ChangeContext, Map, List, List) private void forceCallerAsReviewer(ChangeContext ctx, Map current, List ups, List del) if (current.isEmpty() && ups.isEmpty()) { // TODO Find another way to link reviewers to changes. if (del.isEmpty()) { // If no existing label is being set to 0, hack in the caller // as a reviewer by picking the first server-wide LabelType. PatchSetApproval c = new PatchSetApproval(new PatchSetApproval.Key(psId, user.getAccountId(), ctx.getChangeControl().getLabelTypes().getLabelTypes().get(0).getLabelId()), (short) 0, TimeUtil.nowTs()); c.setGranted(ctx.getWhen()); ups.add(c); } else { // Pick a random label that is about to be deleted and keep it. Iterator i = del.iterator(); PatchSetApproval c = i.next(); c.setValue((short) 0); c.setGranted(ctx.getWhen()); i.remove(); ups.add(c); } }
4874 1242677882GerritCodeReview/gerritDave Borowitz9d12abd177cea3ff708b6b762e1a5a6179a1942d TODO Find another way to link reviewers to changes. TODO Find another way to link reviewers to changes. CLASS_OR_METHOD_CHANGED forceCallerAsReviewer(ChangeContext, Map, List, List) private void forceCallerAsReviewer(ChangeContext ctx, Map current, List ups, List del) if (current.isEmpty() && ups.isEmpty()) { // TODO Find another way to link reviewers to changes. if (del.isEmpty()) { // If no existing label is being set to 0, hack in the caller // as a reviewer by picking the first server-wide LabelType. PatchSetApproval c = new PatchSetApproval(new PatchSetApproval.Key(psId, user.getAccountId(), ctx.getChangeControl().getLabelTypes().getLabelTypes().get(0).getLabelId()), (short) 0, TimeUtil.nowTs()); c.setGranted(ctx.getWhen()); ups.add(c); } else { // Pick a random label that is about to be deleted and keep it. Iterator i = del.iterator(); PatchSetApproval c = i.next(); c.setValue((short) 0); c.setGranted(ctx.getWhen()); i.remove(); ups.add(c); } }
4873 1242677881GerritCodeReview/gerritDave Borowitz9d12abd177cea3ff708b6b762e1a5a6179a1942d TODO Allow updating some labels even when closed. TODO Allow updating some labels even when closed. CLASS_OR_METHOD_CHANGED updateLabels(ChangeContext) private boolean updateLabels(ChangeContext ctx) throws OrmException Map labels = in.labels; if (labels == null) { labels = Collections.emptyMap(); } List del = Lists.newArrayList(); List ups = Lists.newArrayList(); Map current = scanLabels(ctx, del); ChangeUpdate update = ctx.getChangeUpdate(); LabelTypes labelTypes = ctx.getChangeControl().getLabelTypes(); for (Map.Entry ent : labels.entrySet()) { String name = ent.getKey(); LabelType lt = checkNotNull(labelTypes.byLabel(name), name); if (ctx.getChange().getStatus().isClosed()) { // TODO Allow updating some labels even when closed. continue; } PatchSetApproval c = current.remove(lt.getName()); String normName = lt.getName(); if (ent.getValue() == null || ent.getValue() == 0) { // User requested delete of this label. if (c != null) { if (c.getValue() != 0) { addLabelDelta(normName, (short) 0); } del.add(c); update.putApproval(ent.getKey(), (short) 0); } } else if (c != null && c.getValue() != ent.getValue()) { c.setValue(ent.getValue()); c.setGranted(ctx.getWhen()); ups.add(c); addLabelDelta(normName, c.getValue()); categories.put(normName, c.getValue()); update.putApproval(ent.getKey(), ent.getValue()); } else if (c != null && c.getValue() == ent.getValue()) { current.put(normName, c); } else if (c == null) { c = new PatchSetApproval(new PatchSetApproval.Key(psId, user.getAccountId(), lt.getLabelId()), ent.getValue(), TimeUtil.nowTs()); c.setGranted(ctx.getWhen()); ups.add(c); addLabelDelta(normName, c.getValue()); categories.put(normName, c.getValue()); update.putApproval(ent.getKey(), ent.getValue()); } } forceCallerAsReviewer(ctx, current, ups, del); ctx.getDb().patchSetApprovals().delete(del); ctx.getDb().patchSetApprovals().upsert(ups); return !del.isEmpty() || !ups.isEmpty();
4871 1242678093GerritCodeReview/gerritEdwin Kempind034ca8b141d384b9892acc8000edc5cb39aa16e TODO(dborowitz): Move to IndexConfig and use in more places. Account for all commit predicates plus ref, project, status. SATD_REMOVED byCommitsOnBranchNotMerged(Branch.NameKey, List) public Iterable byCommitsOnBranchNotMerged(Branch.NameKey branch, List hashes) throws OrmException Schema schema = schema(indexes); int batchSize; if (schema != null && schema.hasField(ChangeField.EXACT_COMMIT)) { // Account for all commit predicates plus ref, project, status. batchSize = indexConfig.maxTerms() - 3; } else { batchSize = indexConfig.maxPrefixTerms(); } return byCommitsOnBranchNotMerged(schema, branch, hashes, batchSize);
4869 1242678104GerritCodeReview/gerritSaša Živkov250bab05a83fd6c6f2ab83329a8ef77f5b50d608 None TODO(zivkov): handle the case with multiple merge bases SATD_ADDED predicates(Arguments, String, List) private static List> predicates(final Arguments args, String value, List changes) throws OrmException List> changePredicates = Lists.newArrayListWithCapacity(changes.size()); final Provider db = args.db; for (final Change c : changes) { final ChangeDataCache changeDataCache = new ChangeDataCache(c, db, args.changeDataFactory, args.projectCache); List files = listFiles(c, args, changeDataCache); List> filePredicates = Lists.newArrayListWithCapacity(files.size()); for (String file : files) { filePredicates.add(new EqualsPathPredicate(ChangeQueryBuilder.FIELD_PATH, file)); } List> predicatesForOneChange = Lists.newArrayListWithCapacity(5); predicatesForOneChange.add(not(new LegacyChangeIdPredicate(c.getId()))); predicatesForOneChange.add(new ProjectPredicate(c.getProject().get())); predicatesForOneChange.add(new RefPredicate(c.getDest().get())); OperatorPredicate isMerge = new OperatorPredicate(ChangeQueryBuilder.FIELD_MERGE, value) { @Override public boolean match(ChangeData cd) throws OrmException { ObjectId id = ObjectId.fromString(cd.currentPatchSet().getRevision().get()); try (Repository repo = args.repoManager.openRepository(cd.change().getProject()); RevWalk rw = CodeReviewCommit.newRevWalk(repo)) { RevCommit commit = rw.parseCommit(id); return commit.getParentCount() > 1; } catch (IOException e) { throw new IllegalStateException(e); } } @Override public int getCost() { return 2; } }; predicatesForOneChange.add(or(or(filePredicates), isMerge)); predicatesForOneChange.add(new OperatorPredicate(ChangeQueryBuilder.FIELD_CONFLICTS, value) { @Override public boolean match(ChangeData object) throws OrmException { Change otherChange = object.change(); if (otherChange == null) { return false; } if (!otherChange.getDest().equals(c.getDest())) { return false; } SubmitType submitType = getSubmitType(object); if (submitType == null) { return false; } ObjectId other = ObjectId.fromString(object.currentPatchSet().getRevision().get()); ConflictKey conflictsKey = new ConflictKey(changeDataCache.getTestAgainst(), other, submitType, changeDataCache.getProjectState().isUseContentMerge()); Boolean conflicts = args.conflictsCache.getIfPresent(conflictsKey); if (conflicts != null) { return conflicts; } try (Repository repo = args.repoManager.openRepository(otherChange.getProject()); RevWalk rw = CodeReviewCommit.newRevWalk(repo)) { RevFlag canMergeFlag = rw.newFlag("CAN_MERGE"); CodeReviewCommit commit = (CodeReviewCommit) rw.parseCommit(changeDataCache.getTestAgainst()); SubmitStrategy strategy = args.submitStrategyFactory.create(submitType, db.get(), repo, rw, null, canMergeFlag, getAlreadyAccepted(repo, rw, commit), otherChange.getDest()); CodeReviewCommit otherCommit = (CodeReviewCommit) rw.parseCommit(other); otherCommit.add(canMergeFlag); conflicts = !strategy.dryRun(commit, otherCommit); args.conflictsCache.put(conflictsKey, conflicts); return conflicts; } catch (MergeException | NoSuchProjectException | IOException e) { throw new IllegalStateException(e); } } @Override public int getCost() { return 5; } private SubmitType getSubmitType(ChangeData cd) throws OrmException { SubmitTypeRecord r = new SubmitRuleEvaluator(cd).getSubmitType(); if (r.status != SubmitTypeRecord.Status.OK) { return null; } return r.type; } private Set getAlreadyAccepted(Repository repo, RevWalk rw, CodeReviewCommit tip) throws MergeException { Set alreadyAccepted = Sets.newHashSet(); if (tip != null) { alreadyAccepted.add(tip); } try { for (ObjectId id : changeDataCache.getAlreadyAccepted(repo)) { try { alreadyAccepted.add(rw.parseCommit(id)); } catch (IncorrectObjectTypeException iote) { // Not a commit? Skip over it. } } } catch (IOException e) { throw new MergeException("Failed to determine already accepted commits.", e); } return alreadyAccepted; } }); changePredicates.add(and(predicatesForOneChange)); } return changePredicates;
4867 1242678100GerritCodeReview/gerritDave Borowitza495b26800dde44e0e14dca10d4545a155e2c4de TODO(dborowitz): Support async operations? TODO(dborowitz): Support async operations? CLASS_OR_METHOD_CHANGED create(ReviewDb, Project.NameKey, CurrentUser, Timestamp) public BatchUpdate create(ReviewDb db, Project.NameKey project, CurrentUser user, Timestamp when) mp
4865 1242678103GerritCodeReview/gerritDave Borowitzbb0b7f833c1d324f5a1465f9696bae1fc748f7f2 None TODO(dborowitz): This is only public because callers expect validation to happen before updating any refs, and they are still updating refs manually. Make private once we have migrated ref updates into this class. SATD_ADDED validate() public void validate() throws IOException, InvalidChangeOperationException if (validated || validatePolicy == CommitValidators.Policy.NONE) { return; } CommitValidators cv = commitValidatorsFactory.create(refControl, new NoSshInfo(), git); String refName = patchSet.getId().toRefName(); CommitReceivedEvent event = new CommitReceivedEvent(new ReceiveCommand(ObjectId.zeroId(), commit.getId(), refName), refControl.getProjectControl().getProject(), refControl.getRefName(), commit, user); try { switch(validatePolicy) { case RECEIVE_COMMITS: NoteMap rejectCommits = BanCommit.loadRejectCommitsMap(git, revWalk); cv.validateForReceiveCommits(event, rejectCommits); break; case GERRIT: cv.validateForGerritCommits(event); break; case NONE: break; } } catch (CommitValidationException e) { throw new InvalidChangeOperationException(e.getMessage()); } validated = true;
4864 1242678102GerritCodeReview/gerritDave Borowitzc7d6d8ea5cce439e5aee894673eb5ad74b882034 None Only pushing magic branches: allow a valid push certificate even if the key is not ultimately trusted. Assume anyone with Submit permission to the branch is able to verify during review that the code is legitimate. SATD_ADDED isAllowed(CheckResult, Collection) private static boolean isAllowed(CheckResult result, Collection commands) if (onlyMagicBranches(commands)) { // Only pushing magic branches: allow a valid push certificate even if the // key is not ultimately trusted. Assume anyone with Submit permission to // the branch is able to verify during review that the code is legitimate. return result.isOk(); } else { // Directly updating one or more refs: require a trusted key. return result.isTrusted(); }
4863 1242678088GerritCodeReview/gerritDave Borowitz6f58dbe334eb6a6db72c4e63431b1c25b5102325 This is not exactly the key stored in the store, but is equivalent. In particular, it will have a Bouncy Castle version string. The armored stream reader in PublicKeyStore doesn't give us an easy way to extract the original ASCII armor. This is not exactly the key stored in the store, but is equivalent. In particular, it will have a Bouncy Castle version string. The armored stream reader in PublicKeyStore doesn't give us an easy way to extract the original ASCII armor. CLASS_OR_METHOD_CHANGED toJson(PGPPublicKey, CheckResult) public static GpgKeyInfo toJson(PGPPublicKey key, CheckResult checkResult) throws IOException GpgKeyInfo info = new GpgKeyInfo(); if (key != null) { info.id = PublicKeyStore.keyIdToString(key.getKeyID()); info.fingerprint = Fingerprint.toString(key.getFingerprint()); @SuppressWarnings("unchecked") Iterator userIds = key.getUserIDs(); info.userIds = ImmutableList.copyOf(userIds); try (ByteArrayOutputStream out = new ByteArrayOutputStream(4096); ArmoredOutputStream aout = new ArmoredOutputStream(out)) { // This is not exactly the key stored in the store, but is equivalent. In // particular, it will have a Bouncy Castle version string. The armored // stream reader in PublicKeyStore doesn't give us an easy way to extract // the original ASCII armor. key.encode(aout); info.key = new String(out.toByteArray(), UTF_8); } } info.status = checkResult.getStatus(); info.problems = checkResult.getProblems(); return info;
4862 1242678088GerritCodeReview/gerritDave Borowitz36b35b9d71360f4043ebd8d982343a7dcf24abaa This is not exactly the key stored in the store, but is equivalent. In particular, it will have a Bouncy Castle version string. The armored stream reader in PublicKeyStore doesn't give us an easy way to extract the original ASCII armor. This is not exactly the key stored in the store, but is equivalent. In particular, it will have a Bouncy Castle version string. The armored stream reader in PublicKeyStore doesn't give us an easy way to extract the original ASCII armor. CLASS_OR_METHOD_CHANGED toJson(PGPPublicKeyRing, PublicKeyChecker, PublicKeyStore) static GpgKeyInfo toJson(PGPPublicKeyRing keyRing, PublicKeyChecker checker, PublicKeyStore store) throws IOException PGPPublicKey key = keyRing.getPublicKey(); GpgKeyInfo info = new GpgKeyInfo(); info.id = PublicKeyStore.keyIdToString(key.getKeyID()); info.fingerprint = Fingerprint.toString(key.getFingerprint()); @SuppressWarnings("unchecked") Iterator userIds = key.getUserIDs(); info.userIds = ImmutableList.copyOf(userIds); try (ByteArrayOutputStream out = new ByteArrayOutputStream(4096); ArmoredOutputStream aout = new ArmoredOutputStream(out)) { // This is not exactly the key stored in the store, but is equivalent. In // particular, it will have a Bouncy Castle version string. The armored // stream reader in PublicKeyStore doesn't give us an easy way to extract // the original ASCII armor. key.encode(aout); info.key = new String(out.toByteArray(), UTF_8); } CheckResult checkResult = checker.check(key, store); info.status = checkResult.getStatus(); info.problems = checkResult.getProblems(); return info;
4859 1242678096GerritCodeReview/gerritDave Borowitz555ae63f86edf4f8b68a015db6519bdd514310f5 TODO(dborowitz): Not sure this is guaranteed in general. TODO(dborowitz): Not sure this is guaranteed in general. CLASS_OR_METHOD_CHANGED addOp(ChangeControl, Op) public BatchUpdate addOp(ChangeControl ctl, Op op) Change.Id id = ctl.getChange().getId(); ChangeControl old = changeControls.get(id); // TODO(dborowitz): Not sure this is guaranteed in general. checkArgument(old == null || old == ctl, "mismatched ChangeControls for change %s", id); ops.put(id, op); changeControls.put(id, ctl); return this;
4858 1242678100GerritCodeReview/gerritDave Borowitz555ae63f86edf4f8b68a015db6519bdd514310f5 TODO(dborowitz): Document that update contains the old change info. TODO(dborowitz): Support async operations? SATD_CHANGED create(ReviewDb, Project.NameKey, Timestamp) public BatchUpdate create(ReviewDb db, Project.NameKey project, Timestamp when) mp
4855 1242678100GerritCodeReview/gerritDave Borowitz4d31d96633af58da695aa47e5793b0fbb2861383 None TODO(dborowitz): Document that update contains the old change info. SATD_ADDED create(ReviewDb, Project.NameKey, Timestamp) public BatchUpdate create(ReviewDb db, Project.NameKey project, Timestamp when) mp
4851 1242678098GerritCodeReview/gerritDavid Pursehouse7a7fa9cc1cb3fdf6e1e1a603dc06fc06206f0265 None TODO(dborowitz): Kill once callers are migrated. Eventually, callers should always be responsible for executing. SATD_ADDED insert() public Change insert() throws InvalidChangeOperationException, OrmException, IOException, NoSuchChangeException init(); validate(); final AtomicReference updatedChange = new AtomicReference<>(); final AtomicReference> oldReviewers = new AtomicReference<>(); // TODO(dborowitz): Kill once callers are migrated. // Eventually, callers should always be responsible for executing. boolean executeBatch = false; BatchUpdate bu = batchUpdate; if (batchUpdate == null) { bu = batchUpdateFactory.create(db, ctl.getChange().getProject(), patchSet.getCreatedOn()); executeBatch = true; } try { bu.getBatchRefUpdate().addCommand(new ReceiveCommand(ObjectId.zeroId(), commit, patchSet.getRefName(), ReceiveCommand.Type.CREATE)); bu.addChangeOp(new ChangeOp(ctl) { @Override public void call(ReviewDb db, ChangeUpdate update) throws Exception { Change c = db.changes().get(update.getChange().getId()); final PatchSet.Id currentPatchSetId = c.currentPatchSetId(); if (!c.getStatus().isOpen() && !allowClosed) { throw new InvalidChangeOperationException(String.format("Change %s is closed", c.getId())); } ChangeUtil.insertAncestors(db, patchSet.getId(), commit); if (groups != null) { patchSet.setGroups(groups); } else { patchSet.setGroups(GroupCollector.getCurrentGroups(db, c)); } db.patchSets().insert(Collections.singleton(patchSet)); if (sendMail) { oldReviewers.set(approvalsUtil.getReviewers(db, ctl.getNotes())); } updatedChange.set(db.changes().atomicUpdate(c.getId(), new AtomicUpdate() { @Override public Change update(Change change) { if (change.getStatus().isClosed() && !allowClosed) { return null; } if (!change.currentPatchSetId().equals(currentPatchSetId)) { return null; } if (change.getStatus() != Change.Status.DRAFT && !allowClosed) { change.setStatus(Change.Status.NEW); } change.setCurrentPatchSet(patchSetInfoFactory.get(commit, patchSet.getId())); ChangeUtil.updated(change); return change; } })); if (updatedChange.get() == null) { throw new ChangeModifiedException(String.format("Change %s was modified", c.getId())); } approvalCopier.copy(db, ctl, patchSet); if (messageIsForChange()) { cmUtil.addChangeMessage(db, update, changeMessage); } } }); if (!messageIsForChange()) { commitMessageNotForChange(bu); } if (sendMail) { bu.addPostOp(new Callable() { @Override public Void call() { Change c = updatedChange.get(); try { PatchSetInfo info = patchSetInfoFactory.get(commit, patchSet.getId()); ReplacePatchSetSender cm = replacePatchSetFactory.create(c.getId()); cm.setFrom(user.getAccountId()); cm.setPatchSet(patchSet, info); cm.setChangeMessage(changeMessage); cm.addReviewers(oldReviewers.get().get(ReviewerState.REVIEWER)); cm.addExtraCC(oldReviewers.get().get(ReviewerState.CC)); cm.send(); } catch (Exception err) { log.error("Cannot send email for new patch set on change " + c.getId(), err); } return null; } }); } if (runHooks) { bu.addPostOp(new Callable() { @Override public Void call() throws OrmException { hooks.doPatchsetCreatedHook(updatedChange.get(), patchSet, db); return null; } }); } if (executeBatch) { bu.execute(); } } catch (UpdateException e) { Throwables.propagateIfInstanceOf(e.getCause(), NoSuchChangeException.class); Throwables.propagateIfInstanceOf(e.getCause(), InvalidChangeOperationException.class); Throwables.propagateIfInstanceOf(e.getCause(), OrmException.class); Throwables.propagateIfInstanceOf(e.getCause(), IOException.class); Throwables.propagateIfPossible(e.getCause()); throw new OrmException(e); } finally { if (executeBatch) { bu.close(); } } return updatedChange.get();
4850 1242678097GerritCodeReview/gerritDavid Pursehouse7a7fa9cc1cb3fdf6e1e1a603dc06fc06206f0265 None TODO(dborowitz): Support async operations? SATD_ADDED addPostOp(Callable) public BatchUpdate addPostOp(Callable update) postOps.add(update); return this;
4849 1242678096GerritCodeReview/gerritDavid Pursehouse7a7fa9cc1cb3fdf6e1e1a603dc06fc06206f0265 None TODO(dborowitz): Not sure this is guaranteed in general. SATD_ADDED addChangeOp(ChangeOp) public BatchUpdate addChangeOp(ChangeOp op) Change.Id id = op.ctl.getChange().getId(); ChangeControl old = changeControls.get(id); // TODO(dborowitz): Not sure this is guaranteed in general. checkArgument(old == null || old == op.ctl, "mismatched ChangeControls for change %s", id); changeOps.put(id, op); changeControls.put(id, op.ctl); return this;
4848 1242678095GerritCodeReview/gerritDavid Pursehouse7a7fa9cc1cb3fdf6e1e1a603dc06fc06206f0265 None TODO(dborowitz): Manual injection so we can throw IOException from the SATD_ADDED call(ReviewDb, ChangeUpdate) public abstract void call(ReviewDb db, ChangeUpdate update) throws Exception mp
4847 1242678032GerritCodeReview/gerritEdwin Kempin22345d1c2a9a6fa3f84bc100640438c029da6134 TODO(dborowitz): Consider putting ChangeData in CodeReviewCommit. TODO(dborowitz): Consider putting ChangeData in CodeReviewCommit. CLASS_OR_METHOD_CHANGED ChangeData> validateChangeList(Collection) private ListMultimap validateChangeList(Collection submitted) throws MergeException logDebug("Validating {} changes", submitted.size()); ListMultimap toSubmit = ArrayListMultimap.create(); Map allRefs; try { allRefs = repo.getRefDatabase().getRefs(ALL); } catch (IOException e) { throw new MergeException(e.getMessage(), e); } Set tips = new HashSet<>(); for (Ref r : allRefs.values()) { tips.add(r.getObjectId()); } for (ChangeData cd : submitted) { ChangeControl ctl; Change chg; try { ctl = cd.changeControl(); // Reload change in case index was stale. chg = cd.reloadChange(); } catch (OrmException e) { throw new MergeException("Failed to validate changes", e); } Change.Id changeId = cd.getId(); if (chg.getStatus() != Change.Status.NEW) { logDebug("Change {} is not new: {}", changeId, chg.getStatus()); continue; } if (chg.currentPatchSetId() == null) { logError("Missing current patch set on change " + changeId); commits.put(changeId, CodeReviewCommit.noPatchSet(ctl)); continue; } PatchSet ps; Branch.NameKey destBranch = chg.getDest(); try { ps = cd.currentPatchSet(); } catch (OrmException e) { throw new MergeException("Cannot query the database", e); } if (ps == null || ps.getRevision() == null || ps.getRevision().get() == null) { logError("Missing patch set or revision on change " + changeId); commits.put(changeId, CodeReviewCommit.noPatchSet(ctl)); continue; } String idstr = ps.getRevision().get(); ObjectId id; try { id = ObjectId.fromString(idstr); } catch (IllegalArgumentException iae) { logError("Invalid revision on patch set " + ps.getId()); commits.put(changeId, CodeReviewCommit.noPatchSet(ctl)); continue; } if (!tips.contains(id)) { // TODO Technically the proper way to do this test is to use a // RevWalk on "$id --not --all" and test for an empty set. But // that is way slower than looking for a ref directly pointing // at the desired tip. We should always have a ref available. // // TODO this is actually an error, the branch is gone but we // want to merge the issue. We can't safely do that if the // tip is not reachable. // logError("Revision " + idstr + " of patch set " + ps.getId() + " is not contained in any ref"); commits.put(changeId, CodeReviewCommit.revisionGone(ctl)); continue; } CodeReviewCommit commit; try { commit = rw.parseCommit(id); } catch (IOException e) { logError("Invalid commit " + idstr + " on patch set " + ps.getId(), e); commits.put(changeId, CodeReviewCommit.revisionGone(ctl)); continue; } // TODO(dborowitz): Consider putting ChangeData in CodeReviewCommit. commit.setControl(ctl); commit.setPatchsetId(ps.getId()); commits.put(changeId, commit); MergeValidators mergeValidators = mergeValidatorsFactory.create(); try { mergeValidators.validatePreMerge(repo, commit, destProject, destBranch, ps.getId()); } catch (MergeValidationException mve) { logDebug("Revision {} of patch set {} failed validation: {}", idstr, ps.getId(), mve.getStatus()); commit.setStatusCode(mve.getStatus()); continue; } SubmitType submitType; submitType = getSubmitType(commit.getControl(), ps); if (submitType == null) { logError("No submit type for revision " + idstr + " of patch set " + ps.getId()); commit.setStatusCode(CommitMergeStatus.NO_SUBMIT_TYPE); continue; } commit.add(canMergeFlag); toSubmit.put(submitType, cd); } logDebug("Submitting on this run: {}", toSubmit); return toSubmit;
4846 1242677748GerritCodeReview/gerritEdwin Kempin22345d1c2a9a6fa3f84bc100640438c029da6134 TODO Technically the proper way to do this test is to use a RevWalk on \"$id --not --all\" and test for an empty set. But that is way slower than looking for a ref directly pointing at the desired tip. We should always have a ref available. TODO this is actually an error, the branch is gone but we want to merge the issue. We can't safely do that if the tip is not reachable. TODO Technically the proper way to do this test is to use a RevWalk on \"$id --not --all\" and test for an empty set. But that is way slower than looking for a ref directly pointing at the desired tip. We should always have a ref available. TODO this is actually an error, the branch is gone but we want to merge the issue. We can't safely do that if the tip is not reachable. CLASS_OR_METHOD_CHANGED ChangeData> validateChangeList(Collection) private ListMultimap validateChangeList(Collection submitted) throws MergeException logDebug("Validating {} changes", submitted.size()); ListMultimap toSubmit = ArrayListMultimap.create(); Map allRefs; try { allRefs = repo.getRefDatabase().getRefs(ALL); } catch (IOException e) { throw new MergeException(e.getMessage(), e); } Set tips = new HashSet<>(); for (Ref r : allRefs.values()) { tips.add(r.getObjectId()); } for (ChangeData cd : submitted) { ChangeControl ctl; Change chg; try { ctl = cd.changeControl(); // Reload change in case index was stale. chg = cd.reloadChange(); } catch (OrmException e) { throw new MergeException("Failed to validate changes", e); } Change.Id changeId = cd.getId(); if (chg.getStatus() != Change.Status.NEW) { logDebug("Change {} is not new: {}", changeId, chg.getStatus()); continue; } if (chg.currentPatchSetId() == null) { logError("Missing current patch set on change " + changeId); commits.put(changeId, CodeReviewCommit.noPatchSet(ctl)); continue; } PatchSet ps; Branch.NameKey destBranch = chg.getDest(); try { ps = cd.currentPatchSet(); } catch (OrmException e) { throw new MergeException("Cannot query the database", e); } if (ps == null || ps.getRevision() == null || ps.getRevision().get() == null) { logError("Missing patch set or revision on change " + changeId); commits.put(changeId, CodeReviewCommit.noPatchSet(ctl)); continue; } String idstr = ps.getRevision().get(); ObjectId id; try { id = ObjectId.fromString(idstr); } catch (IllegalArgumentException iae) { logError("Invalid revision on patch set " + ps.getId()); commits.put(changeId, CodeReviewCommit.noPatchSet(ctl)); continue; } if (!tips.contains(id)) { // TODO Technically the proper way to do this test is to use a // RevWalk on "$id --not --all" and test for an empty set. But // that is way slower than looking for a ref directly pointing // at the desired tip. We should always have a ref available. // // TODO this is actually an error, the branch is gone but we // want to merge the issue. We can't safely do that if the // tip is not reachable. // logError("Revision " + idstr + " of patch set " + ps.getId() + " is not contained in any ref"); commits.put(changeId, CodeReviewCommit.revisionGone(ctl)); continue; } CodeReviewCommit commit; try { commit = rw.parseCommit(id); } catch (IOException e) { logError("Invalid commit " + idstr + " on patch set " + ps.getId(), e); commits.put(changeId, CodeReviewCommit.revisionGone(ctl)); continue; } // TODO(dborowitz): Consider putting ChangeData in CodeReviewCommit. commit.setControl(ctl); commit.setPatchsetId(ps.getId()); commits.put(changeId, commit); MergeValidators mergeValidators = mergeValidatorsFactory.create(); try { mergeValidators.validatePreMerge(repo, commit, destProject, destBranch, ps.getId()); } catch (MergeValidationException mve) { logDebug("Revision {} of patch set {} failed validation: {}", idstr, ps.getId(), mve.getStatus()); commit.setStatusCode(mve.getStatus()); continue; } SubmitType submitType; submitType = getSubmitType(commit.getControl(), ps); if (submitType == null) { logError("No submit type for revision " + idstr + " of patch set " + ps.getId()); commit.setStatusCode(CommitMergeStatus.NO_SUBMIT_TYPE); continue; } commit.add(canMergeFlag); toSubmit.put(submitType, cd); } logDebug("Submitting on this run: {}", toSubmit); return toSubmit;
4844 1242678061GerritCodeReview/gerritHugo Arès947b5e53ac75f8085fd9dd5c3bd05dbf284ee288 TODO(dborowitz): Merge this with AcceptanceTestRequestScope. TODO(dborowitz): Merge this with AcceptanceTestRequestScope. FILE_PATH_CHANGED module() static Module module() return new AbstractModule() { @Override public void configure() { install(new GerritRequestModule()); bind(RequestScopePropagator.class).to(Propagator.class); bindScope(RequestScoped.class, InProcessProtocol.REQUEST); } @Provides @RemotePeer SocketAddress getSocketAddress() { // TODO(dborowitz): Could potentially fake this with thread ID or // something. throw new OutOfScopeException("No remote peer in acceptance tests"); } };
4843 1242678060GerritCodeReview/gerritHugo Arès947b5e53ac75f8085fd9dd5c3bd05dbf284ee288 TODO(dborowitz): Could potentially fake this with thread ID or something. TODO(dborowitz): Could potentially fake this with thread ID or something. FILE_PATH_CHANGED configure() public void configure() install(new GerritRequestModule()); bind(RequestScopePropagator.class).to(Propagator.class); bindScope(RequestScoped.class, InProcessProtocol.REQUEST);
4841 1242678093GerritCodeReview/gerritDave Borowitz5702eaa254c6e24626a2fffdc609ad9810c152ac None TODO(dborowitz): Move to IndexConfig and use in more places. SATD_ADDED byCommitsOnBranchNotMerged(Branch.NameKey, List) public Iterable byCommitsOnBranchNotMerged(Branch.NameKey branch, List hashes) throws OrmException Schema schema = schema(indexes); int batchSize; if (schema != null && schema.hasField(ChangeField.EXACT_COMMIT)) { // TODO(dborowitz): Move to IndexConfig and use in more places. batchSize = 500; } else { batchSize = indexConfig.maxPrefixTerms(); } return byCommitsOnBranchNotMerged(schema, branch, hashes, batchSize);
4840 1242678092GerritCodeReview/gerritDavid Ostrovskyc4e01806ffef77aadaa8b9805b4129067ec0bcdd None TODO(davido): Find a better way to prevent key maps collisions SATD_ADDED registerKeys() public void registerKeys() super.registerKeys(); KeyMap localKeyMap = KeyMap.create(); localKeyMap.on("Ctrl-L", gotoLine()).on("Cmd-L", gotoLine()).on("Cmd-S", save()); // TODO(davido): Find a better way to prevent key maps collisions if (prefs.keyMapType() != KeyMapType.EMACS) { localKeyMap.on("Ctrl-S", save()); } cm.addKeyMap(localKeyMap);
4839 1242678091GerritCodeReview/gerritDave Borowitz8e289571372266874c84f755cfac21919964162c None TODO(dborowitz): Require self certification. SATD_ADDED checkWebOfTrust(PGPPublicKey, PublicKeyStore, int, Set) private CheckResult checkWebOfTrust(PGPPublicKey key, PublicKeyStore store, int depth, Set seen) if (trusted == null || store == null) { // Trust checking not configured. return CheckResult.OK; } Fingerprint fp = new Fingerprint(key.getFingerprint()); if (seen.contains(fp)) { return new CheckResult("Key is trusted in a cycle"); } seen.add(fp); Fingerprint trustedFp = trusted.get(key.getKeyID()); if (trustedFp != null && trustedFp.equals(fp)) { // Directly trusted. return CheckResult.OK; } else if (depth >= maxTrustDepth) { return new CheckResult("No path of depth <= " + maxTrustDepth + " to a trusted key"); } List signerResults = new ArrayList<>(); @SuppressWarnings("unchecked") Iterator userIds = key.getUserIDs(); while (userIds.hasNext()) { String userId = userIds.next(); @SuppressWarnings("unchecked") Iterator sigs = key.getSignaturesForID(userId); while (sigs.hasNext()) { PGPSignature sig = sigs.next(); // TODO(dborowitz): Handle CERTIFICATION_REVOCATION. if (sig.getSignatureType() != PGPSignature.DEFAULT_CERTIFICATION && sig.getSignatureType() != PGPSignature.POSITIVE_CERTIFICATION) { // Not a certification. continue; } PGPPublicKey signer = getSigner(store, sig, userId, key, signerResults); // TODO(dborowitz): Require self certification. if (signer == null || Arrays.equals(signer.getFingerprint(), key.getFingerprint())) { continue; } CheckResult signerResult = checkTrustSubpacket(sig, depth); if (signerResult.isOk()) { signerResult = check(signer, store, depth + 1, false, seen); if (signerResult.isOk()) { return CheckResult.OK; } } signerResults.add(new CheckResult("Certification by " + keyToString(signer) + " is valid, but key is not trusted")); } } List problems = new ArrayList<>(); problems.add("No path to a trusted key"); for (CheckResult signerResult : signerResults) { problems.addAll(signerResult.getProblems()); } return new CheckResult(problems);
4838 1242678090GerritCodeReview/gerritDave Borowitz8e289571372266874c84f755cfac21919964162c None TODO(dborowitz): Handle CERTIFICATION_REVOCATION. SATD_ADDED checkWebOfTrust(PGPPublicKey, PublicKeyStore, int, Set) private CheckResult checkWebOfTrust(PGPPublicKey key, PublicKeyStore store, int depth, Set seen) if (trusted == null || store == null) { // Trust checking not configured. return CheckResult.OK; } Fingerprint fp = new Fingerprint(key.getFingerprint()); if (seen.contains(fp)) { return new CheckResult("Key is trusted in a cycle"); } seen.add(fp); Fingerprint trustedFp = trusted.get(key.getKeyID()); if (trustedFp != null && trustedFp.equals(fp)) { // Directly trusted. return CheckResult.OK; } else if (depth >= maxTrustDepth) { return new CheckResult("No path of depth <= " + maxTrustDepth + " to a trusted key"); } List signerResults = new ArrayList<>(); @SuppressWarnings("unchecked") Iterator userIds = key.getUserIDs(); while (userIds.hasNext()) { String userId = userIds.next(); @SuppressWarnings("unchecked") Iterator sigs = key.getSignaturesForID(userId); while (sigs.hasNext()) { PGPSignature sig = sigs.next(); // TODO(dborowitz): Handle CERTIFICATION_REVOCATION. if (sig.getSignatureType() != PGPSignature.DEFAULT_CERTIFICATION && sig.getSignatureType() != PGPSignature.POSITIVE_CERTIFICATION) { // Not a certification. continue; } PGPPublicKey signer = getSigner(store, sig, userId, key, signerResults); // TODO(dborowitz): Require self certification. if (signer == null || Arrays.equals(signer.getFingerprint(), key.getFingerprint())) { continue; } CheckResult signerResult = checkTrustSubpacket(sig, depth); if (signerResult.isOk()) { signerResult = check(signer, store, depth + 1, false, seen); if (signerResult.isOk()) { return CheckResult.OK; } } signerResults.add(new CheckResult("Certification by " + keyToString(signer) + " is valid, but key is not trusted")); } } List problems = new ArrayList<>(); problems.add("No path to a trusted key"); for (CheckResult signerResult : signerResults) { problems.addAll(signerResult.getProblems()); } return new CheckResult(problems);
4837 1242678089GerritCodeReview/gerritDavid Pursehouse00f97a67714d5b716d6b501273a8ccbd78b8ec5c None TODO(dborowitz): Swap out GitRepositoryManager somehow? Will probably be necessary for notedb anyway. SATD_ADDED prepopulatedFields() public void prepopulatedFields() throws Exception assume().that(notesMigration.enabled()).isFalse(); TestRepository repo = createProject("repo"); Change change = newChange(repo, null, null, null, null).insert(); db = new DisabledReviewDb(); requestContext.setContext(newRequestContext(userId)); // Use QueryProcessor directly instead of API so we get ChangeDatas back. List cds = queryProcessor.queryChanges(queryBuilder.parse(change.getId().toString())).changes(); assertThat(cds).hasSize(1); ChangeData cd = cds.get(0); cd.change(); cd.patchSets(); cd.currentApprovals(); cd.changedLines(); cd.reviewedBy(); // TODO(dborowitz): Swap out GitRepositoryManager somehow? Will probably be // necessary for notedb anyway. cd.isMergeable(); // Don't use ExpectedException since that wouldn't distinguish between // failures here and on the previous calls. try { cd.messages(); } catch (AssertionError e) { assertThat(e.getMessage()).isEqualTo(DisabledReviewDb.MESSAGE); }
4836 1242678088GerritCodeReview/gerritDave Borowitzb8336f176afb28c2b7bdf193571df932ea819321 This is not exactly the key stored in the store, but is equivalent. In particular, it will have a Bouncy Castle version string. The armored stream reader in PublicKeyStore doesn't give us an easy way to extract the original ASCII armor. This is not exactly the key stored in the store, but is equivalent. In particular, it will have a Bouncy Castle version string. The armored stream reader in PublicKeyStore doesn't give us an easy way to extract the original ASCII armor. FILE_PATH_CHANGED toJson(PGPPublicKeyRing) static GpgKeyInfo toJson(PGPPublicKeyRing keyRing) throws IOException PGPPublicKey key = keyRing.getPublicKey(); GpgKeyInfo info = new GpgKeyInfo(); info.id = PublicKeyStore.keyIdToString(key.getKeyID()); info.fingerprint = Fingerprint.toString(key.getFingerprint()); @SuppressWarnings("unchecked") Iterator userIds = key.getUserIDs(); info.userIds = ImmutableList.copyOf(userIds); try (ByteArrayOutputStream out = new ByteArrayOutputStream(4096); ArmoredOutputStream aout = new ArmoredOutputStream(out)) { // This is not exactly the key stored in the store, but is equivalent. In // particular, it will have a Bouncy Castle version string. The armored // stream reader in PublicKeyStore doesn't give us an easy way to extract // the original ASCII armor. key.encode(aout); info.key = new String(out.toByteArray(), UTF_8); } return info;
4835 1242678087GerritCodeReview/gerritDave Borowitzb8336f176afb28c2b7bdf193571df932ea819321 TODO(dborowitz): Backoff and retry on LOCK_FAILURE. TODO(dborowitz): Backoff and retry on LOCK_FAILURE. FILE_PATH_CHANGED storeKeys(AccountResource, List, Set) private void storeKeys(AccountResource rsrc, List keyRings, Set toRemove) throws BadRequestException, ResourceConflictException, PGPException, IOException try (PublicKeyStore store = storeProvider.get()) { for (PGPPublicKeyRing keyRing : keyRings) { PGPPublicKey key = keyRing.getPublicKey(); CheckResult result = checker.check(key); if (!result.isOk()) { throw new BadRequestException(String.format("Problems with public key %s:\n%s", keyToString(key), Joiner.on('\n').join(result.getProblems()))); } store.add(keyRing); } for (Fingerprint fp : toRemove) { store.remove(fp.get()); } CommitBuilder cb = new CommitBuilder(); PersonIdent committer = serverIdent.get(); cb.setAuthor(rsrc.getUser().newCommitterIdent(committer.getWhen(), committer.getTimeZone())); cb.setCommitter(committer); RefUpdate.Result saveResult = store.save(cb); switch(saveResult) { case NEW: case FAST_FORWARD: case FORCED: case NO_CHANGE: break; default: // TODO(dborowitz): Backoff and retry on LOCK_FAILURE. throw new ResourceConflictException("Failed to save public keys: " + saveResult); } }
4833 1242678087GerritCodeReview/gerritDave Borowitzeab3aff03aac39a0c236aeb77f5a316f5bee097f TODO(dborowitz): Backoff and retry on LOCK_FAILURE. TODO(dborowitz): Backoff and retry on LOCK_FAILURE. CLASS_OR_METHOD_CHANGED storeKeys(AccountResource, List, Set) private void storeKeys(AccountResource rsrc, List keyRings, Set toRemove) throws BadRequestException, ResourceConflictException, PGPException, IOException try (PublicKeyStore store = storeProvider.get()) { for (PGPPublicKeyRing keyRing : keyRings) { PGPPublicKey key = keyRing.getPublicKey(); CheckResult result = checker.check(key); if (!result.isOk()) { throw new BadRequestException(String.format("Problems with public key %s:\n%s", keyToString(key), Joiner.on('\n').join(result.getProblems()))); } store.add(keyRing); } for (Fingerprint fp : toRemove) { store.remove(fp.get()); } CommitBuilder cb = new CommitBuilder(); PersonIdent committer = serverIdent.get(); cb.setAuthor(rsrc.getUser().newCommitterIdent(committer.getWhen(), committer.getTimeZone())); cb.setCommitter(committer); RefUpdate.Result saveResult = store.save(cb); switch(saveResult) { case NEW: case FAST_FORWARD: case FORCED: case NO_CHANGE: break; default: // TODO(dborowitz): Backoff and retry on LOCK_FAILURE. throw new ResourceConflictException("Failed to save public keys: " + saveResult); } }
4831 1242678088GerritCodeReview/gerritDave Borowitzed170f35f6bb3d8ec2e1fb4709bf119401ab1233 None This is not exactly the key stored in the store, but is equivalent. In particular, it will have a Bouncy Castle version string. The armored stream reader in PublicKeyStore doesn't give us an easy way to extract the original ASCII armor. SATD_ADDED toJson(PGPPublicKeyRing) static GpgKeyInfo toJson(PGPPublicKeyRing keyRing) throws IOException PGPPublicKey key = keyRing.getPublicKey(); GpgKeyInfo info = new GpgKeyInfo(); info.id = PublicKeyStore.keyIdToString(key.getKeyID()); info.fingerprint = PublicKeyStore.fingerprintToString(key.getFingerprint()); @SuppressWarnings("unchecked") Iterator userIds = key.getUserIDs(); info.userIds = ImmutableList.copyOf(userIds); try (ByteArrayOutputStream out = new ByteArrayOutputStream(4096); ArmoredOutputStream aout = new ArmoredOutputStream(out)) { // This is not exactly the key stored in the store, but is equivalent. In // particular, it will have a Bouncy Castle version string. The armored // stream reader in PublicKeyStore doesn't give us an easy way to extract // the original ASCII armor. key.encode(aout); info.key = new String(out.toByteArray(), UTF_8); } return info;
4830 1242678087GerritCodeReview/gerritDave Borowitzed170f35f6bb3d8ec2e1fb4709bf119401ab1233 None TODO(dborowitz): Backoff and retry on LOCK_FAILURE. SATD_ADDED storeKeys(AccountResource, List) private void storeKeys(AccountResource rsrc, List keyRings) throws BadRequestException, ResourceConflictException, PGPException, IOException try (PublicKeyStore store = storeProvider.get()) { for (PGPPublicKeyRing keyRing : keyRings) { PGPPublicKey key = keyRing.getPublicKey(); CheckResult result = checker.check(key); if (!result.isOk()) { throw new BadRequestException(String.format("Problems with public key %s:\n%s", keyToString(key), Joiner.on('\n').join(result.getProblems()))); } store.add(keyRing); } CommitBuilder cb = new CommitBuilder(); PersonIdent committer = serverIdent.get(); cb.setAuthor(rsrc.getUser().newCommitterIdent(committer.getWhen(), committer.getTimeZone())); cb.setCommitter(committer); RefUpdate.Result saveResult = store.save(cb); switch(saveResult) { case NEW: case FAST_FORWARD: case FORCED: break; default: // TODO(dborowitz): Backoff and retry on LOCK_FAILURE. throw new ResourceConflictException("Failed to save public key: " + saveResult); } }
4829 1242678068GerritCodeReview/gerritShawn Pearcec9fa09eb5e253a1df25f295df3894ff6797bd57f None If the container didn't do the authentication we might have done it in the front-end web server. Try to split the identity out of the Authorization header and honor it. SATD_ADDED getRemoteUser(HttpServletRequest, String) public static String getRemoteUser(HttpServletRequest req, String loginHeader) if (AUTHORIZATION.equals(loginHeader)) { String user = emptyToNull(req.getRemoteUser()); if (user != null) { // The container performed the authentication, and has the user // identity already decoded for us. Honor that as we have been // configured to honor HTTP authentication. return user; } // If the container didn't do the authentication we might // have done it in the front-end web server. Try to split // the identity out of the Authorization header and honor it. String auth = req.getHeader(AUTHORIZATION); return extractUsername(auth); } else { // Nonstandard HTTP header. We have been told to trust this // header blindly as-is. return emptyToNull(req.getHeader(loginHeader)); }
4827 1242677814GerritCodeReview/gerritDave Borowitzfdbb76233c34c23f446e11f97047ef75c84a4704 We sometimes collapsed an edit together in a strange way, such that the edges of each text is identical. Fix by by dropping out that incorrectly replaced region. We sometimes collapsed an edit together in a strange way, such that the edges of each text is identical. Fix by by dropping out that incorrectly replaced region. CLASS_OR_METHOD_CHANGED compute(Text, Text, List) static IntraLineDiff compute(Text aText, Text bText, List edits) throws Exception combineLineEdits(edits, aText, bText); for (int i = 0; i < edits.size(); i++) { Edit e = edits.get(i); if (e.getType() == Edit.Type.REPLACE) { CharText a = new CharText(aText, e.getBeginA(), e.getEndA()); CharText b = new CharText(bText, e.getBeginB(), e.getEndB()); CharTextComparator cmp = new CharTextComparator(); List wordEdits = MyersDiff.INSTANCE.diff(cmp, a, b); // Combine edits that are really close together. If they are // just a few characters apart we tend to get better results // by joining them together and taking the whole span. // for (int j = 0; j < wordEdits.size() - 1; ) { Edit c = wordEdits.get(j); Edit n = wordEdits.get(j + 1); if (n.getBeginA() - c.getEndA() <= 5 || n.getBeginB() - c.getEndB() <= 5) { int ab = c.getBeginA(); int ae = n.getEndA(); int bb = c.getBeginB(); int be = n.getEndB(); if (canCoalesce(a, c.getEndA(), n.getBeginA()) && canCoalesce(b, c.getEndB(), n.getBeginB())) { wordEdits.set(j, new Edit(ab, ae, bb, be)); wordEdits.remove(j + 1); continue; } } j++; } // Apply some simple rules to fix up some of the edits. Our // logic above, along with our per-character difference tends // to produce some crazy stuff. // for (int j = 0; j < wordEdits.size(); j++) { Edit c = wordEdits.get(j); int ab = c.getBeginA(); int ae = c.getEndA(); int bb = c.getBeginB(); int be = c.getEndB(); // Sometimes the diff generator produces an INSERT or DELETE // right up against a REPLACE, but we only find this after // we've also played some shifting games on the prior edit. // If that happened to us, coalesce them together so we can // correct this mess for the user. If we don't we wind up // with silly stuff like "es" -> "es = Addresses". // if (1 < j) { Edit p = wordEdits.get(j - 1); if (p.getEndA() == ab || p.getEndB() == bb) { if (p.getEndA() == ab && p.getBeginA() < p.getEndA()) { ab = p.getBeginA(); } if (p.getEndB() == bb && p.getBeginB() < p.getEndB()) { bb = p.getBeginB(); } wordEdits.remove(--j); } } // We sometimes collapsed an edit together in a strange way, // such that the edges of each text is identical. Fix by // by dropping out that incorrectly replaced region. // while (ab < ae && bb < be && cmp.equals(a, ab, b, bb)) { ab++; bb++; } while (ab < ae && bb < be && cmp.equals(a, ae - 1, b, be - 1)) { ae--; be--; } // The leading part of an edit and its trailing part in the same // text might be identical. Slide down that edit and use the tail // rather than the leading bit. If however the edit is only on a // whitespace block try to shift it to the left margin, assuming // that it is an indentation change. // boolean aShift = true; if (ab < ae && isOnlyWhitespace(a, ab, ae)) { int lf = findLF(wordEdits, j, a, ab); if (lf < ab && a.charAt(lf) == '\n') { int nb = lf + 1; int p = 0; while (p < ae - ab) { if (cmp.equals(a, ab + p, a, ab + p)) { p++; } else { break; } } if (p == ae - ab) { ab = nb; ae = nb + p; aShift = false; } } } if (aShift) { while (0 < ab && ab < ae && a.charAt(ab - 1) != '\n' && cmp.equals(a, ab - 1, a, ae - 1)) { ab--; ae--; } if (!a.isLineStart(ab) || !a.contains(ab, ae, '\n')) { while (ab < ae && ae < a.size() && cmp.equals(a, ab, a, ae)) { ab++; ae++; if (a.charAt(ae - 1) == '\n') { break; } } } } boolean bShift = true; if (bb < be && isOnlyWhitespace(b, bb, be)) { int lf = findLF(wordEdits, j, b, bb); if (lf < bb && b.charAt(lf) == '\n') { int nb = lf + 1; int p = 0; while (p < be - bb) { if (cmp.equals(b, bb + p, b, bb + p)) { p++; } else { break; } } if (p == be - bb) { bb = nb; be = nb + p; bShift = false; } } } if (bShift) { while (0 < bb && bb < be && b.charAt(bb - 1) != '\n' && cmp.equals(b, bb - 1, b, be - 1)) { bb--; be--; } if (!b.isLineStart(bb) || !b.contains(bb, be, '\n')) { while (bb < be && be < b.size() && cmp.equals(b, bb, b, be)) { bb++; be++; if (b.charAt(be - 1) == '\n') { break; } } } } // If most of a line was modified except the LF was common, make // the LF part of the modification region. This is easier to read. // if (// ab < ae && // (ab == 0 || a.charAt(ab - 1) == '\n') && ae < a.size() && a.charAt(ae - 1) != '\n' && a.charAt(ae) == '\n') { ae++; } if (// bb < be && // (bb == 0 || b.charAt(bb - 1) == '\n') && be < b.size() && b.charAt(be - 1) != '\n' && b.charAt(be) == '\n') { be++; } wordEdits.set(j, new Edit(ab, ae, bb, be)); } edits.set(i, new ReplaceEdit(e, wordEdits)); } } return new IntraLineDiff(edits);
4826 1242677813GerritCodeReview/gerritDave Borowitzfdbb76233c34c23f446e11f97047ef75c84a4704 Apply some simple rules to fix up some of the edits. Our logic above, along with our per-character difference tends to produce some crazy stuff. Apply some simple rules to fix up some of the edits. Our logic above, along with our per-character difference tends to produce some crazy stuff. CLASS_OR_METHOD_CHANGED compute(Text, Text, List) static IntraLineDiff compute(Text aText, Text bText, List edits) throws Exception combineLineEdits(edits, aText, bText); for (int i = 0; i < edits.size(); i++) { Edit e = edits.get(i); if (e.getType() == Edit.Type.REPLACE) { CharText a = new CharText(aText, e.getBeginA(), e.getEndA()); CharText b = new CharText(bText, e.getBeginB(), e.getEndB()); CharTextComparator cmp = new CharTextComparator(); List wordEdits = MyersDiff.INSTANCE.diff(cmp, a, b); // Combine edits that are really close together. If they are // just a few characters apart we tend to get better results // by joining them together and taking the whole span. // for (int j = 0; j < wordEdits.size() - 1; ) { Edit c = wordEdits.get(j); Edit n = wordEdits.get(j + 1); if (n.getBeginA() - c.getEndA() <= 5 || n.getBeginB() - c.getEndB() <= 5) { int ab = c.getBeginA(); int ae = n.getEndA(); int bb = c.getBeginB(); int be = n.getEndB(); if (canCoalesce(a, c.getEndA(), n.getBeginA()) && canCoalesce(b, c.getEndB(), n.getBeginB())) { wordEdits.set(j, new Edit(ab, ae, bb, be)); wordEdits.remove(j + 1); continue; } } j++; } // Apply some simple rules to fix up some of the edits. Our // logic above, along with our per-character difference tends // to produce some crazy stuff. // for (int j = 0; j < wordEdits.size(); j++) { Edit c = wordEdits.get(j); int ab = c.getBeginA(); int ae = c.getEndA(); int bb = c.getBeginB(); int be = c.getEndB(); // Sometimes the diff generator produces an INSERT or DELETE // right up against a REPLACE, but we only find this after // we've also played some shifting games on the prior edit. // If that happened to us, coalesce them together so we can // correct this mess for the user. If we don't we wind up // with silly stuff like "es" -> "es = Addresses". // if (1 < j) { Edit p = wordEdits.get(j - 1); if (p.getEndA() == ab || p.getEndB() == bb) { if (p.getEndA() == ab && p.getBeginA() < p.getEndA()) { ab = p.getBeginA(); } if (p.getEndB() == bb && p.getBeginB() < p.getEndB()) { bb = p.getBeginB(); } wordEdits.remove(--j); } } // We sometimes collapsed an edit together in a strange way, // such that the edges of each text is identical. Fix by // by dropping out that incorrectly replaced region. // while (ab < ae && bb < be && cmp.equals(a, ab, b, bb)) { ab++; bb++; } while (ab < ae && bb < be && cmp.equals(a, ae - 1, b, be - 1)) { ae--; be--; } // The leading part of an edit and its trailing part in the same // text might be identical. Slide down that edit and use the tail // rather than the leading bit. If however the edit is only on a // whitespace block try to shift it to the left margin, assuming // that it is an indentation change. // boolean aShift = true; if (ab < ae && isOnlyWhitespace(a, ab, ae)) { int lf = findLF(wordEdits, j, a, ab); if (lf < ab && a.charAt(lf) == '\n') { int nb = lf + 1; int p = 0; while (p < ae - ab) { if (cmp.equals(a, ab + p, a, ab + p)) { p++; } else { break; } } if (p == ae - ab) { ab = nb; ae = nb + p; aShift = false; } } } if (aShift) { while (0 < ab && ab < ae && a.charAt(ab - 1) != '\n' && cmp.equals(a, ab - 1, a, ae - 1)) { ab--; ae--; } if (!a.isLineStart(ab) || !a.contains(ab, ae, '\n')) { while (ab < ae && ae < a.size() && cmp.equals(a, ab, a, ae)) { ab++; ae++; if (a.charAt(ae - 1) == '\n') { break; } } } } boolean bShift = true; if (bb < be && isOnlyWhitespace(b, bb, be)) { int lf = findLF(wordEdits, j, b, bb); if (lf < bb && b.charAt(lf) == '\n') { int nb = lf + 1; int p = 0; while (p < be - bb) { if (cmp.equals(b, bb + p, b, bb + p)) { p++; } else { break; } } if (p == be - bb) { bb = nb; be = nb + p; bShift = false; } } } if (bShift) { while (0 < bb && bb < be && b.charAt(bb - 1) != '\n' && cmp.equals(b, bb - 1, b, be - 1)) { bb--; be--; } if (!b.isLineStart(bb) || !b.contains(bb, be, '\n')) { while (bb < be && be < b.size() && cmp.equals(b, bb, b, be)) { bb++; be++; if (b.charAt(be - 1) == '\n') { break; } } } } // If most of a line was modified except the LF was common, make // the LF part of the modification region. This is easier to read. // if (// ab < ae && // (ab == 0 || a.charAt(ab - 1) == '\n') && ae < a.size() && a.charAt(ae - 1) != '\n' && a.charAt(ae) == '\n') { ae++; } if (// bb < be && // (bb == 0 || b.charAt(bb - 1) == '\n') && be < b.size() && b.charAt(be - 1) != '\n' && b.charAt(be) == '\n') { be++; } wordEdits.set(j, new Edit(ab, ae, bb, be)); } edits.set(i, new ReplaceEdit(e, wordEdits)); } } return new IntraLineDiff(edits);
4825 1242677812GerritCodeReview/gerritDave Borowitzfdbb76233c34c23f446e11f97047ef75c84a4704 Combine edits that are really close together. If they are just a few characters apart we tend to get better results by joining them together and taking the whole span. Combine edits that are really close together. If they are just a few characters apart we tend to get better results by joining them together and taking the whole span. CLASS_OR_METHOD_CHANGED compute(Text, Text, List) static IntraLineDiff compute(Text aText, Text bText, List edits) throws Exception combineLineEdits(edits, aText, bText); for (int i = 0; i < edits.size(); i++) { Edit e = edits.get(i); if (e.getType() == Edit.Type.REPLACE) { CharText a = new CharText(aText, e.getBeginA(), e.getEndA()); CharText b = new CharText(bText, e.getBeginB(), e.getEndB()); CharTextComparator cmp = new CharTextComparator(); List wordEdits = MyersDiff.INSTANCE.diff(cmp, a, b); // Combine edits that are really close together. If they are // just a few characters apart we tend to get better results // by joining them together and taking the whole span. // for (int j = 0; j < wordEdits.size() - 1; ) { Edit c = wordEdits.get(j); Edit n = wordEdits.get(j + 1); if (n.getBeginA() - c.getEndA() <= 5 || n.getBeginB() - c.getEndB() <= 5) { int ab = c.getBeginA(); int ae = n.getEndA(); int bb = c.getBeginB(); int be = n.getEndB(); if (canCoalesce(a, c.getEndA(), n.getBeginA()) && canCoalesce(b, c.getEndB(), n.getBeginB())) { wordEdits.set(j, new Edit(ab, ae, bb, be)); wordEdits.remove(j + 1); continue; } } j++; } // Apply some simple rules to fix up some of the edits. Our // logic above, along with our per-character difference tends // to produce some crazy stuff. // for (int j = 0; j < wordEdits.size(); j++) { Edit c = wordEdits.get(j); int ab = c.getBeginA(); int ae = c.getEndA(); int bb = c.getBeginB(); int be = c.getEndB(); // Sometimes the diff generator produces an INSERT or DELETE // right up against a REPLACE, but we only find this after // we've also played some shifting games on the prior edit. // If that happened to us, coalesce them together so we can // correct this mess for the user. If we don't we wind up // with silly stuff like "es" -> "es = Addresses". // if (1 < j) { Edit p = wordEdits.get(j - 1); if (p.getEndA() == ab || p.getEndB() == bb) { if (p.getEndA() == ab && p.getBeginA() < p.getEndA()) { ab = p.getBeginA(); } if (p.getEndB() == bb && p.getBeginB() < p.getEndB()) { bb = p.getBeginB(); } wordEdits.remove(--j); } } // We sometimes collapsed an edit together in a strange way, // such that the edges of each text is identical. Fix by // by dropping out that incorrectly replaced region. // while (ab < ae && bb < be && cmp.equals(a, ab, b, bb)) { ab++; bb++; } while (ab < ae && bb < be && cmp.equals(a, ae - 1, b, be - 1)) { ae--; be--; } // The leading part of an edit and its trailing part in the same // text might be identical. Slide down that edit and use the tail // rather than the leading bit. If however the edit is only on a // whitespace block try to shift it to the left margin, assuming // that it is an indentation change. // boolean aShift = true; if (ab < ae && isOnlyWhitespace(a, ab, ae)) { int lf = findLF(wordEdits, j, a, ab); if (lf < ab && a.charAt(lf) == '\n') { int nb = lf + 1; int p = 0; while (p < ae - ab) { if (cmp.equals(a, ab + p, a, ab + p)) { p++; } else { break; } } if (p == ae - ab) { ab = nb; ae = nb + p; aShift = false; } } } if (aShift) { while (0 < ab && ab < ae && a.charAt(ab - 1) != '\n' && cmp.equals(a, ab - 1, a, ae - 1)) { ab--; ae--; } if (!a.isLineStart(ab) || !a.contains(ab, ae, '\n')) { while (ab < ae && ae < a.size() && cmp.equals(a, ab, a, ae)) { ab++; ae++; if (a.charAt(ae - 1) == '\n') { break; } } } } boolean bShift = true; if (bb < be && isOnlyWhitespace(b, bb, be)) { int lf = findLF(wordEdits, j, b, bb); if (lf < bb && b.charAt(lf) == '\n') { int nb = lf + 1; int p = 0; while (p < be - bb) { if (cmp.equals(b, bb + p, b, bb + p)) { p++; } else { break; } } if (p == be - bb) { bb = nb; be = nb + p; bShift = false; } } } if (bShift) { while (0 < bb && bb < be && b.charAt(bb - 1) != '\n' && cmp.equals(b, bb - 1, b, be - 1)) { bb--; be--; } if (!b.isLineStart(bb) || !b.contains(bb, be, '\n')) { while (bb < be && be < b.size() && cmp.equals(b, bb, b, be)) { bb++; be++; if (b.charAt(be - 1) == '\n') { break; } } } } // If most of a line was modified except the LF was common, make // the LF part of the modification region. This is easier to read. // if (// ab < ae && // (ab == 0 || a.charAt(ab - 1) == '\n') && ae < a.size() && a.charAt(ae - 1) != '\n' && a.charAt(ae) == '\n') { ae++; } if (// bb < be && // (bb == 0 || b.charAt(bb - 1) == '\n') && be < b.size() && b.charAt(be - 1) != '\n' && b.charAt(be) == '\n') { be++; } wordEdits.set(j, new Edit(ab, ae, bb, be)); } edits.set(i, new ReplaceEdit(e, wordEdits)); } } return new IntraLineDiff(edits);
4820 1242678085GerritCodeReview/gerritShawn Pearceba24004f70d7e1407010d0884b0133780de1b12a We can only approximately reconstruct what the submit rule evaluator would have done. These should really come from a stored submit record. We can only approximately reconstruct what the submit rule evaluator would have done. These should really come from a stored submit record. CLASS_OR_METHOD_CHANGED addOption(ListChangesOption) public ChangeJson addOption(ListChangesOption o) options.add(o); return this;
4819 1242678037GerritCodeReview/gerritShawn Pearce2efa6030fd7ae5d86da97f09da4aa2077751f717 TODO(sbeller): why do we need to treat followup specially here? The followup action is a client-side only operation that does not have a server side handler. It must be manually registered into the resulting action map. SATD_REMOVED ActionInfo> toActionMap(ChangeControl) private Map toActionMap(ChangeControl ctl) Map out = new LinkedHashMap<>(); if (!ctl.getCurrentUser().isIdentifiedUser()) { return out; } Provider userProvider = Providers.of(ctl.getCurrentUser()); for (UiAction.Description d : UiActions.from(changeViews, new ChangeResource(ctl), userProvider)) { out.put(d.getId(), new ActionInfo(d)); } // The followup action is a client-side only operation that does not // have a server side handler. It must be manually registered into the // resulting action map. if (ctl.getChange().getStatus().isOpen()) { UiAction.Description descr = new UiAction.Description(); PrivateInternals_UiActionDescription.setId(descr, "followup"); PrivateInternals_UiActionDescription.setMethod(descr, "POST"); descr.setTitle("Create follow-up change"); out.put(descr.getId(), new ActionInfo(descr)); } return out;
4817 1242678032GerritCodeReview/gerritStefan Beller7063a4b8f0659d8b22bb62a349872e5350fb6a7b TODO(dborowitz): Consider putting ChangeData in CodeReviewCommit. TODO(dborowitz): Consider putting ChangeData in CodeReviewCommit. CLASS_OR_METHOD_CHANGED ChangeData> validateChangeList(List) private ListMultimap validateChangeList(List submitted) throws MergeException logDebug("Validating {} changes", submitted.size()); ListMultimap toSubmit = ArrayListMultimap.create(); Map allRefs; try { allRefs = repo.getRefDatabase().getRefs(ALL); } catch (IOException e) { throw new MergeException(e.getMessage(), e); } Set tips = new HashSet<>(); for (Ref r : allRefs.values()) { tips.add(r.getObjectId()); } for (ChangeData cd : submitted) { ChangeControl ctl; Change chg; try { ctl = cd.changeControl(); // Reload change in case index was stale. chg = cd.reloadChange(); } catch (OrmException e) { throw new MergeException("Failed to validate changes", e); } Change.Id changeId = cd.getId(); if (chg.getStatus() != Change.Status.SUBMITTED && chg.getStatus() != Change.Status.NEW) { logDebug("Change {} is not new or submitted: {}", changeId, chg.getStatus()); continue; } if (chg.currentPatchSetId() == null) { logError("Missing current patch set on change " + changeId); commits.put(changeId, CodeReviewCommit.noPatchSet(ctl)); continue; } PatchSet ps; Branch.NameKey destBranch = chg.getDest(); try { ps = cd.currentPatchSet(); } catch (OrmException e) { throw new MergeException("Cannot query the database", e); } if (ps == null || ps.getRevision() == null || ps.getRevision().get() == null) { logError("Missing patch set or revision on change " + changeId); commits.put(changeId, CodeReviewCommit.noPatchSet(ctl)); continue; } String idstr = ps.getRevision().get(); ObjectId id; try { id = ObjectId.fromString(idstr); } catch (IllegalArgumentException iae) { logError("Invalid revision on patch set " + ps.getId()); commits.put(changeId, CodeReviewCommit.noPatchSet(ctl)); continue; } if (!tips.contains(id)) { // TODO Technically the proper way to do this test is to use a // RevWalk on "$id --not --all" and test for an empty set. But // that is way slower than looking for a ref directly pointing // at the desired tip. We should always have a ref available. // // TODO this is actually an error, the branch is gone but we // want to merge the issue. We can't safely do that if the // tip is not reachable. // logError("Revision " + idstr + " of patch set " + ps.getId() + " is not contained in any ref"); commits.put(changeId, CodeReviewCommit.revisionGone(ctl)); continue; } CodeReviewCommit commit; try { commit = (CodeReviewCommit) rw.parseCommit(id); } catch (IOException e) { logError("Invalid commit " + idstr + " on patch set " + ps.getId(), e); commits.put(changeId, CodeReviewCommit.revisionGone(ctl)); continue; } // TODO(dborowitz): Consider putting ChangeData in CodeReviewCommit. commit.setControl(ctl); commit.setPatchsetId(ps.getId()); commits.put(changeId, commit); MergeValidators mergeValidators = mergeValidatorsFactory.create(); try { mergeValidators.validatePreMerge(repo, commit, destProject, destBranch, ps.getId()); } catch (MergeValidationException mve) { logDebug("Revision {} of patch set {} failed validation: {}", idstr, ps.getId(), mve.getStatus()); commit.setStatusCode(mve.getStatus()); continue; } SubmitType submitType; submitType = getSubmitType(commit.getControl(), ps); if (submitType == null) { logError("No submit type for revision " + idstr + " of patch set " + ps.getId()); commit.setStatusCode(CommitMergeStatus.NO_SUBMIT_TYPE); continue; } commit.add(canMergeFlag); toSubmit.put(submitType, cd); } logDebug("Submitting on this run: {}", toSubmit); return toSubmit;
4816 1242677748GerritCodeReview/gerritStefan Beller7063a4b8f0659d8b22bb62a349872e5350fb6a7b TODO Technically the proper way to do this test is to use a RevWalk on \"$id --not --all\" and test for an empty set. But that is way slower than looking for a ref directly pointing at the desired tip. We should always have a ref available. TODO this is actually an error, the branch is gone but we want to merge the issue. We can't safely do that if the tip is not reachable. TODO Technically the proper way to do this test is to use a RevWalk on \"$id --not --all\" and test for an empty set. But that is way slower than looking for a ref directly pointing at the desired tip. We should always have a ref available. TODO this is actually an error, the branch is gone but we want to merge the issue. We can't safely do that if the tip is not reachable. CLASS_OR_METHOD_CHANGED ChangeData> validateChangeList(List) private ListMultimap validateChangeList(List submitted) throws MergeException logDebug("Validating {} changes", submitted.size()); ListMultimap toSubmit = ArrayListMultimap.create(); Map allRefs; try { allRefs = repo.getRefDatabase().getRefs(ALL); } catch (IOException e) { throw new MergeException(e.getMessage(), e); } Set tips = new HashSet<>(); for (Ref r : allRefs.values()) { tips.add(r.getObjectId()); } for (ChangeData cd : submitted) { ChangeControl ctl; Change chg; try { ctl = cd.changeControl(); // Reload change in case index was stale. chg = cd.reloadChange(); } catch (OrmException e) { throw new MergeException("Failed to validate changes", e); } Change.Id changeId = cd.getId(); if (chg.getStatus() != Change.Status.SUBMITTED && chg.getStatus() != Change.Status.NEW) { logDebug("Change {} is not new or submitted: {}", changeId, chg.getStatus()); continue; } if (chg.currentPatchSetId() == null) { logError("Missing current patch set on change " + changeId); commits.put(changeId, CodeReviewCommit.noPatchSet(ctl)); continue; } PatchSet ps; Branch.NameKey destBranch = chg.getDest(); try { ps = cd.currentPatchSet(); } catch (OrmException e) { throw new MergeException("Cannot query the database", e); } if (ps == null || ps.getRevision() == null || ps.getRevision().get() == null) { logError("Missing patch set or revision on change " + changeId); commits.put(changeId, CodeReviewCommit.noPatchSet(ctl)); continue; } String idstr = ps.getRevision().get(); ObjectId id; try { id = ObjectId.fromString(idstr); } catch (IllegalArgumentException iae) { logError("Invalid revision on patch set " + ps.getId()); commits.put(changeId, CodeReviewCommit.noPatchSet(ctl)); continue; } if (!tips.contains(id)) { // TODO Technically the proper way to do this test is to use a // RevWalk on "$id --not --all" and test for an empty set. But // that is way slower than looking for a ref directly pointing // at the desired tip. We should always have a ref available. // // TODO this is actually an error, the branch is gone but we // want to merge the issue. We can't safely do that if the // tip is not reachable. // logError("Revision " + idstr + " of patch set " + ps.getId() + " is not contained in any ref"); commits.put(changeId, CodeReviewCommit.revisionGone(ctl)); continue; } CodeReviewCommit commit; try { commit = (CodeReviewCommit) rw.parseCommit(id); } catch (IOException e) { logError("Invalid commit " + idstr + " on patch set " + ps.getId(), e); commits.put(changeId, CodeReviewCommit.revisionGone(ctl)); continue; } // TODO(dborowitz): Consider putting ChangeData in CodeReviewCommit. commit.setControl(ctl); commit.setPatchsetId(ps.getId()); commits.put(changeId, commit); MergeValidators mergeValidators = mergeValidatorsFactory.create(); try { mergeValidators.validatePreMerge(repo, commit, destProject, destBranch, ps.getId()); } catch (MergeValidationException mve) { logDebug("Revision {} of patch set {} failed validation: {}", idstr, ps.getId(), mve.getStatus()); commit.setStatusCode(mve.getStatus()); continue; } SubmitType submitType; submitType = getSubmitType(commit.getControl(), ps); if (submitType == null) { logError("No submit type for revision " + idstr + " of patch set " + ps.getId()); commit.setStatusCode(CommitMergeStatus.NO_SUBMIT_TYPE); continue; } commit.add(canMergeFlag); toSubmit.put(submitType, cd); } logDebug("Submitting on this run: {}", toSubmit); return toSubmit;
4815 1242678084GerritCodeReview/gerritDave Borowitz8792bd67d75fcff96acdf0656bf841b532eb75ce None TODO: Check the committer, too. SATD_ADDED submitWithCherryPick() public void submitWithCherryPick() throws Exception RevCommit initialHead = getRemoteHead(); PushOneCommit.Result change = createChange("Change 1", "a.txt", "content"); submit(change.getChangeId()); RevCommit oldHead = getRemoteHead(); testRepo.reset(initialHead); PushOneCommit.Result change2 = createChange("Change 2", "b.txt", "other content"); submit(change2.getChangeId()); assertCherryPick(testRepo, false); RevCommit newHead = getRemoteHead(); assertThat(newHead.getParentCount()).isEqualTo(1); assertThat(newHead.getParent(0)).isEqualTo(oldHead); assertCurrentRevision(change2.getChangeId(), 2, newHead); assertSubmitter(change2.getChangeId(), 1); assertSubmitter(change2.getChangeId(), 2); assertPersonEquals(admin.getIdent(), newHead.getAuthorIdent()); // TODO: Check the committer, too. // assertPersonEquals(admin.getIdent(), newHead.getCommitterIdent());
4812 1242678083GerritCodeReview/gerritDave Borowitz61003e99cee563caf0668045c9989047fc919c0b None TODO(dborowitz): Support certifications by other trusted keys? SATD_ADDED checkCertifications(PGPPublicKey, String, List) private static void checkCertifications(PGPPublicKey key, String userId, List problems) @SuppressWarnings("unchecked") Iterator sigs = key.getSignaturesForID(userId); if (sigs == null) { sigs = Collections.emptyIterator(); } boolean ok = false; boolean revoked = false; try { while (sigs.hasNext()) { PGPSignature sig = sigs.next(); if (sig.getKeyID() != key.getKeyID()) { // TODO(dborowitz): Support certifications by other trusted keys? continue; } else if (sig.getSignatureType() != DEFAULT_CERTIFICATION && sig.getSignatureType() != POSITIVE_CERTIFICATION && sig.getSignatureType() != CERTIFICATION_REVOCATION) { continue; } sig.init(new BcPGPContentVerifierBuilderProvider(), key); if (sig.verifyCertification(userId, key)) { if (sig.getSignatureType() == CERTIFICATION_REVOCATION) { revoked = true; } else { ok = true; } } else { problems.add("Invalid signature for User ID " + userId); } } } catch (PGPException e) { problems.add("Error in certifications"); log.warn("Error in certification verification for public key: " + keyToString(key), e); } if (revoked) { problems.add("User ID " + userId + " is revoked"); } else if (!ok) { problems.add("No certification for User ID " + userId); }
4811 1242678082GerritCodeReview/gerritDave Borowitz61003e99cee563caf0668045c9989047fc919c0b None TODO(dborowitz): Remove some/all of these checks. SATD_ADDED checkCertifications(PGPPublicKey, String, List) private static void checkCertifications(PGPPublicKey key, String userId, List problems) @SuppressWarnings("unchecked") Iterator sigs = key.getSignaturesForID(userId); if (sigs == null) { sigs = Collections.emptyIterator(); } boolean ok = false; boolean revoked = false; try { while (sigs.hasNext()) { PGPSignature sig = sigs.next(); if (sig.getKeyID() != key.getKeyID()) { // TODO(dborowitz): Support certifications by other trusted keys? continue; } else if (sig.getSignatureType() != DEFAULT_CERTIFICATION && sig.getSignatureType() != POSITIVE_CERTIFICATION && sig.getSignatureType() != CERTIFICATION_REVOCATION) { continue; } sig.init(new BcPGPContentVerifierBuilderProvider(), key); if (sig.verifyCertification(userId, key)) { if (sig.getSignatureType() == CERTIFICATION_REVOCATION) { revoked = true; } else { ok = true; } } else { problems.add("Invalid signature for User ID " + userId); } } } catch (PGPException e) { problems.add("Error in certifications"); log.warn("Error in certification verification for public key: " + keyToString(key), e); } if (revoked) { problems.add("User ID " + userId + " is revoked"); } else if (!ok) { problems.add("No certification for User ID " + userId); }
4810 1242678081GerritCodeReview/gerritDave Borowitz61003e99cee563caf0668045c9989047fc919c0b None TODO(dborowitz): put method. SATD_ADDED empty() private static PGPPublicKeyRingCollection empty() throws PGPException, IOException return new PGPPublicKeyRingCollection(Collections.emptyList());
4806 1242677840GerritCodeReview/gerritDave Borowitz7a45eb305cb24ac2c50f5a39e985ef205bdbf107 TODO since this is performed \"in the background\" no mail will be sent to inform users about the updated branch TODO since this is performed \"in the background\" no mail will be sent to inform users about the updated branch CLASS_OR_METHOD_CHANGED updateGitlinks(ReviewDb, Branch.NameKey, Collection) private void updateGitlinks(ReviewDb db, Branch.NameKey subscriber, Collection updates) throws SubmoduleException PersonIdent author = null; Repository pdb = null; RevWalk recRw = null; StringBuilder msgbuf = new StringBuilder("Updated git submodules\n\n"); try { boolean sameAuthorForAll = true; pdb = repoManager.openRepository(subscriber.getParentKey()); if (pdb.getRef(subscriber.get()) == null) { throw new SubmoduleException("The branch was probably deleted from the subscriber repository"); } DirCache dc = readTree(pdb, pdb.getRef(subscriber.get())); DirCacheEditor ed = dc.editor(); for (SubmoduleSubscription s : updates) { try (Repository subrepo = repoManager.openRepository(s.getSubmodule().getParentKey()); RevWalk rw = CodeReviewCommit.newRevWalk(subrepo)) { Ref ref = subrepo.getRefDatabase().exactRef(s.getSubmodule().get()); if (ref == null) { ed.add(new DeletePath(s.getPath())); continue; } final ObjectId updateTo = ref.getObjectId(); RevCommit newCommit = rw.parseCommit(updateTo); if (author == null) { author = newCommit.getAuthorIdent(); } else if (!author.equals(newCommit.getAuthorIdent())) { sameAuthorForAll = false; } DirCacheEntry dce = dc.getEntry(s.getPath()); ObjectId oldId = null; if (dce != null) { if (!dce.getFileMode().equals(FileMode.GITLINK)) { log.error("Requested to update gitlink " + s.getPath() + " in " + s.getSubmodule().getParentKey().get() + " but entry " + "doesn't have gitlink file mode."); continue; } oldId = dce.getObjectId(); } else { // This submodule did not exist before. We do not want to add // the full submodule history to the commit message, so omit it. oldId = updateTo; } ed.add(new PathEdit(s.getPath()) { @Override public void apply(DirCacheEntry ent) { ent.setFileMode(FileMode.GITLINK); ent.setObjectId(updateTo); } }); msgbuf.append("Project: " + s.getSubmodule().getParentKey().get()); msgbuf.append(" " + s.getSubmodule().getShortName()); msgbuf.append(" " + updateTo.getName()); msgbuf.append("\n\n"); try { rw.markStart(newCommit); if (oldId != null) { rw.markUninteresting(rw.parseCommit(oldId)); } for (RevCommit c : rw) { msgbuf.append(c.getFullMessage() + "\n\n"); } } catch (IOException e) { logAndThrowSubmoduleException("Could not perform a revwalk to " + "create superproject commit message", e); } } } ed.finish(); if (!sameAuthorForAll || author == null) { author = myIdent; } ObjectInserter oi = pdb.newObjectInserter(); ObjectId tree = dc.writeTree(oi); ObjectId currentCommitId = pdb.getRef(subscriber.get()).getObjectId(); CommitBuilder commit = new CommitBuilder(); commit.setTreeId(tree); commit.setParentIds(new ObjectId[] { currentCommitId }); commit.setAuthor(author); commit.setCommitter(myIdent); commit.setMessage(msgbuf.toString()); oi.insert(commit); oi.flush(); ObjectId commitId = oi.idFor(Constants.OBJ_COMMIT, commit.build()); final RefUpdate rfu = pdb.updateRef(subscriber.get()); rfu.setForceUpdate(false); rfu.setNewObjectId(commitId); rfu.setExpectedOldObjectId(currentCommitId); rfu.setRefLogMessage("Submit to " + subscriber.getParentKey().get(), true); switch(rfu.update()) { case NEW: case FAST_FORWARD: gitRefUpdated.fire(subscriber.getParentKey(), rfu); changeHooks.doRefUpdatedHook(subscriber, rfu, account); // TODO since this is performed "in the background" no mail will be // sent to inform users about the updated branch break; default: throw new IOException(rfu.getResult().name()); } recRw = new RevWalk(pdb); // Recursive call: update subscribers of the subscriber updateSuperProjects(db, Sets.newHashSet(subscriber)); } catch (IOException e) { throw new SubmoduleException("Cannot update gitlinks for " + subscriber.get(), e); } finally { if (recRw != null) { recRw.close(); } if (pdb != null) { pdb.close(); } }
4805 1242678080GerritCodeReview/gerritStefan Beller09feaaccbc81f1089297cb6484052027012895c9 None TODO(sbeller): show only on latest revision SATD_ADDED css() mp RelatedChangesCss css()
4803 1242677994GerritCodeReview/gerritStefan Bellerc5993446ebe1e1627b42bb44555c4635aca3bb26 TODO(yyonas): atomic change is not propagated. TODO(yyonas): atomic change is not propagated. CLASS_OR_METHOD_CHANGED setNew(ChangeNotes, ChangeMessage) private void setNew(ChangeNotes notes, final ChangeMessage msg) throws NoSuchChangeException, IOException Change c = notes.getChange(); Change change = null; ChangeUpdate update = null; try { db.changes().beginTransaction(c.getId()); try { change = db.changes().atomicUpdate(c.getId(), new AtomicUpdate() { @Override public Change update(Change c) { if (c.getStatus().isOpen()) { c.setStatus(Change.Status.NEW); ChangeUtil.updated(c); } return c; } }); ChangeControl control = changeControl(change); // TODO(yyonas): atomic change is not propagated. update = updateFactory.create(control, c.getLastUpdatedOn()); if (msg != null) { cmUtil.addChangeMessage(db, update, msg); } db.commit(); } finally { db.rollback(); } } catch (OrmException err) { logWarn("Cannot record merge failure message", err); } if (update != null) { update.commit(); } indexer.index(db, change); PatchSetApproval submitter = null; try { submitter = approvalsUtil.getSubmitter(db, notes, notes.getChange().currentPatchSetId()); } catch (Exception e) { logError("Cannot get submitter for change " + notes.getChangeId(), e); } if (submitter != null) { try { hooks.doMergeFailedHook(c, accountCache.get(submitter.getAccountId()).getAccount(), db.patchSets().get(c.currentPatchSetId()), msg.getMessage(), db); } catch (OrmException ex) { logError("Cannot run hook for merge failed " + c.getId(), ex); } }
4799 1242677991GerritCodeReview/gerritEdwin Kempine326a1dada8a2705a33f50921dc2b921a4ce3917 TODO(yyonas): atomic update was not propagated TODO(yyonas): atomic update was not propagated CLASS_OR_METHOD_CHANGED abandon(ChangeControl, String, Account) public Change abandon(ChangeControl control, String msgTxt, Account acc) throws ResourceConflictException, OrmException, IOException Change change; ChangeMessage message; ChangeUpdate update; Change.Id changeId = control.getChange().getId(); ReviewDb db = dbProvider.get(); db.changes().beginTransaction(changeId); try { change = db.changes().atomicUpdate(changeId, new AtomicUpdate() { @Override public Change update(Change change) { if (change.getStatus().isOpen()) { change.setStatus(Change.Status.ABANDONED); ChangeUtil.updated(change); return change; } return null; } }); if (change == null) { throw new ResourceConflictException("change is " + status(db.changes().get(changeId))); } // TODO(yyonas): atomic update was not propagated update = updateFactory.create(control, change.getLastUpdatedOn()); message = newMessage(msgTxt, acc != null ? acc.getId() : null, change); cmUtil.addChangeMessage(db, update, message); db.commit(); } finally { db.rollback(); } update.commit(); indexer.index(db, change); try { ReplyToChangeSender cm = abandonedSenderFactory.create(change.getId()); if (acc != null) { cm.setFrom(acc.getId()); } cm.setChangeMessage(message); cm.send(); } catch (Exception e) { log.error("Cannot email update for change " + change.getChangeId(), e); } hooks.doChangeAbandonedHook(change, acc, db.patchSets().get(change.currentPatchSetId()), Strings.emptyToNull(msgTxt), db); return change;
4798 1242678078GerritCodeReview/gerritDave Borowitz90ed9414c2d4e0f8ec60d5e7c0d1463b30f2eee8 None TODO(dborowitz): Support certifications by other trusted keys? SATD_ADDED verifyPublicKeyCertifications(PGPPublicKey, PushCertificateIdent) private boolean verifyPublicKeyCertifications(PGPPublicKey key, PushCertificateIdent ident) @SuppressWarnings("unchecked") Iterator sigs = key.getSignaturesForID(ident.getUserId()); if (sigs == null) { sigs = Collections.emptyIterator(); } boolean valid = false; boolean revoked = false; try { while (sigs.hasNext()) { PGPSignature sig = sigs.next(); if (sig.getKeyID() != key.getKeyID()) { // TODO(dborowitz): Support certifications by other trusted keys? continue; } else if (sig.getSignatureType() != DEFAULT_CERTIFICATION && sig.getSignatureType() != POSITIVE_CERTIFICATION && sig.getSignatureType() != CERTIFICATION_REVOCATION) { continue; } sig.init(new BcPGPContentVerifierBuilderProvider(), key); if (sig.verifyCertification(ident.getUserId(), key)) { if (sig.getSignatureType() == CERTIFICATION_REVOCATION) { revoked = true; } else { valid = true; } } else { log.warn("Invalid signature for pusher identity {} in key: {}", ident.getUserId(), toString(key)); } } } catch (PGPException e) { log.warn("Error in signature verification for public key", e); } if (revoked) { log.warn("Pusher identity {} is revoked in key {}", ident.getUserId(), toString(key)); return false; } else if (!valid) { log.warn("Key does not contain valid certification for pusher identity {}: {}", ident.getUserId(), toString(key)); return false; } return true;
4797 1242678077GerritCodeReview/gerritDave Borowitz90ed9414c2d4e0f8ec60d5e7c0d1463b30f2eee8 TODO(dborowitz): Try all keys. TODO(dborowitz): Try all keys. CLASS_OR_METHOD_CHANGED readPublicKey(long, PushCertificateIdent) private PGPPublicKey readPublicKey(long keyId, PushCertificateIdent expectedIdent) throws IOException try (Repository repo = repoManager.openRepository(allUsers); RevWalk rw = new RevWalk(repo)) { Ref ref = repo.getRefDatabase().exactRef(RefNames.REFS_GPG_KEYS); if (ref == null) { return null; } NoteMap notes = NoteMap.read(rw.getObjectReader(), rw.parseCommit(ref.getObjectId())); Note note = notes.getNote(keyObjectId(keyId)); if (note == null) { return null; } try (InputStream objIn = rw.getObjectReader().open(note.getData(), OBJ_BLOB).openStream(); ArmoredInputStream in = new ArmoredInputStream(objIn)) { PGPObjectFactory factory = new BcPGPObjectFactory(in); PGPPublicKey matched = null; Object obj; while ((obj = factory.nextObject()) != null) { if (!(obj instanceof PGPPublicKeyRing)) { // TODO(dborowitz): Support assertions signed by a trusted key. log.info("Ignoring {} packet in {}", obj.getClass().getSimpleName(), note.getName()); continue; } PGPPublicKeyRing keyRing = (PGPPublicKeyRing) obj; PGPPublicKey key = keyRing.getPublicKey(keyId); if (key == null) { log.warn("Public key ring in {} does not contain key ID {}", note.getName(), keyObjectId(keyId)); continue; } if (matched != null) { // TODO(dborowitz): Try all keys. log.warn("Ignoring key with duplicate ID: {}", toString(key)); continue; } if (!verifyPublicKey(key, expectedIdent)) { continue; } matched = key; } return matched; } }
4796 1242678076GerritCodeReview/gerritDave Borowitz90ed9414c2d4e0f8ec60d5e7c0d1463b30f2eee8 TODO(dborowitz): Support assertions signed by a trusted key. TODO(dborowitz): Support assertions signed by a trusted key. CLASS_OR_METHOD_CHANGED readPublicKey(long, PushCertificateIdent) private PGPPublicKey readPublicKey(long keyId, PushCertificateIdent expectedIdent) throws IOException try (Repository repo = repoManager.openRepository(allUsers); RevWalk rw = new RevWalk(repo)) { Ref ref = repo.getRefDatabase().exactRef(RefNames.REFS_GPG_KEYS); if (ref == null) { return null; } NoteMap notes = NoteMap.read(rw.getObjectReader(), rw.parseCommit(ref.getObjectId())); Note note = notes.getNote(keyObjectId(keyId)); if (note == null) { return null; } try (InputStream objIn = rw.getObjectReader().open(note.getData(), OBJ_BLOB).openStream(); ArmoredInputStream in = new ArmoredInputStream(objIn)) { PGPObjectFactory factory = new BcPGPObjectFactory(in); PGPPublicKey matched = null; Object obj; while ((obj = factory.nextObject()) != null) { if (!(obj instanceof PGPPublicKeyRing)) { // TODO(dborowitz): Support assertions signed by a trusted key. log.info("Ignoring {} packet in {}", obj.getClass().getSimpleName(), note.getName()); continue; } PGPPublicKeyRing keyRing = (PGPPublicKeyRing) obj; PGPPublicKey key = keyRing.getPublicKey(keyId); if (key == null) { log.warn("Public key ring in {} does not contain key ID {}", note.getName(), keyObjectId(keyId)); continue; } if (matched != null) { // TODO(dborowitz): Try all keys. log.warn("Ignoring key with duplicate ID: {}", toString(key)); continue; } if (!verifyPublicKey(key, expectedIdent)) { continue; } matched = key; } return matched; } }
4795 1242678077GerritCodeReview/gerritDave Borowitzc5861d81cb404b5b6a103a139767c2efecaf0ccc None TODO(dborowitz): Try all keys. SATD_ADDED readPublicKey(long) private PGPPublicKey readPublicKey(long keyId) throws IOException try (Repository repo = repoManager.openRepository(allUsers); RevWalk rw = new RevWalk(repo)) { Ref ref = repo.getRefDatabase().exactRef(RefNames.REFS_GPG_KEYS); if (ref == null) { return null; } NoteMap notes = NoteMap.read(rw.getObjectReader(), rw.parseCommit(ref.getObjectId())); Note note = notes.getNote(keyObjectId(keyId)); if (note == null) { return null; } try (InputStream objIn = rw.getObjectReader().open(note.getData(), OBJ_BLOB).openStream(); ArmoredInputStream in = new ArmoredInputStream(objIn)) { PGPObjectFactory factory = new BcPGPObjectFactory(in); PGPPublicKey matched = null; Object obj; while ((obj = factory.nextObject()) != null) { if (!(obj instanceof PGPPublicKeyRing)) { // TODO(dborowitz): Support assertions signed by a trusted key. log.info("Ignoring {} packet in {}", obj.getClass().getSimpleName(), note.getName()); continue; } PGPPublicKeyRing keyRing = (PGPPublicKeyRing) obj; PGPPublicKey key = keyRing.getPublicKey(keyId); if (key == null) { log.warn("Public key ring in {} does not contain key ID {}", note.getName(), keyObjectId(keyId)); continue; } if (matched != null) { // TODO(dborowitz): Try all keys. log.warn("Ignoring key with duplicate ID: {}", toString(key)); continue; } matched = key; } // TODO(dborowitz): Additional key verification, at least user ID // signature. return matched; } }
4794 1242678076GerritCodeReview/gerritDave Borowitzc5861d81cb404b5b6a103a139767c2efecaf0ccc None TODO(dborowitz): Support assertions signed by a trusted key. SATD_ADDED readPublicKey(long) private PGPPublicKey readPublicKey(long keyId) throws IOException try (Repository repo = repoManager.openRepository(allUsers); RevWalk rw = new RevWalk(repo)) { Ref ref = repo.getRefDatabase().exactRef(RefNames.REFS_GPG_KEYS); if (ref == null) { return null; } NoteMap notes = NoteMap.read(rw.getObjectReader(), rw.parseCommit(ref.getObjectId())); Note note = notes.getNote(keyObjectId(keyId)); if (note == null) { return null; } try (InputStream objIn = rw.getObjectReader().open(note.getData(), OBJ_BLOB).openStream(); ArmoredInputStream in = new ArmoredInputStream(objIn)) { PGPObjectFactory factory = new BcPGPObjectFactory(in); PGPPublicKey matched = null; Object obj; while ((obj = factory.nextObject()) != null) { if (!(obj instanceof PGPPublicKeyRing)) { // TODO(dborowitz): Support assertions signed by a trusted key. log.info("Ignoring {} packet in {}", obj.getClass().getSimpleName(), note.getName()); continue; } PGPPublicKeyRing keyRing = (PGPPublicKeyRing) obj; PGPPublicKey key = keyRing.getPublicKey(keyId); if (key == null) { log.warn("Public key ring in {} does not contain key ID {}", note.getName(), keyObjectId(keyId)); continue; } if (matched != null) { // TODO(dborowitz): Try all keys. log.warn("Ignoring key with duplicate ID: {}", toString(key)); continue; } matched = key; } // TODO(dborowitz): Additional key verification, at least user ID // signature. return matched; } }
4792 1242677776GerritCodeReview/gerritDave Borowitza3d67882125065c86ae55ccbb45e36d2ca6e453e The identity information reported about the connection by a RFC 1413 [11] request to the remote agent, if available. Servers MAY choose not to support this feature, or not to request the data for efficiency reasons. \"REMOTE_IDENT\" => \"NYI\" The identity information reported about the connection by a RFC 1413 [11] request to the remote agent, if available. Servers MAY choose not to support this feature, or not to request the data for efficiency reasons. \"REMOTE_IDENT\" => \"NYI\" FILE_PATH_CHANGED makeSiteConfig(SitePaths, Config, SshInfo) private void makeSiteConfig(SitePaths site, Config cfg, SshInfo sshInfo) throws IOException if (!Files.exists(site.tmp_dir)) { Files.createDirectories(site.tmp_dir); } Path myconf = Files.createTempFile(site.tmp_dir, "gitweb_config", ".perl"); // To make our configuration file only readable or writable by us; // this reduces the chances of someone tampering with the file. // // TODO(dborowitz): Is there a portable way to do this with NIO? File myconfFile = myconf.toFile(); myconfFile.setWritable(false, false); myconfFile.setReadable(false, false); myconfFile.setExecutable(false, false); myconfFile.setWritable(true, true); myconfFile.setReadable(true, true); myconfFile.deleteOnExit(); _env.set("GIT_DIR", "."); _env.set("GITWEB_CONFIG", myconf.toAbsolutePath().toString()); try (PrintWriter p = new PrintWriter(Files.newBufferedWriter(myconf, UTF_8))) { p.print("# Autogenerated by Gerrit Code Review \n"); p.print("# DO NOT EDIT\n"); p.print("\n"); // We are mounted at the same level in the context as the main // UI, so we can include the same header and footer scheme. // Path hdr = site.site_header; if (Files.isRegularFile(hdr)) { p.print("$site_header = " + quoteForPerl(hdr) + ";\n"); } Path ftr = site.site_footer; if (Files.isRegularFile(ftr)) { p.print("$site_footer = " + quoteForPerl(ftr) + ";\n"); } // Top level should return to Gerrit's UI. // p.print("$home_link = $ENV{'GERRIT_CONTEXT_PATH'};\n"); p.print("$home_link_str = 'Code Review';\n"); p.print("$favicon = 'favicon.ico';\n"); p.print("$logo = 'gitweb-logo.png';\n"); p.print("$javascript = 'gitweb.js';\n"); p.print("@stylesheets = ('gitweb-default.css');\n"); Path css = site.site_css; if (Files.isRegularFile(css)) { p.print("push @stylesheets, 'gitweb-site.css';\n"); } // Try to make the title match Gerrit's normal window title // scheme of host followed by 'Code Review'. // p.print("$site_name = $home_link_str;\n"); p.print("$site_name = qq{$1 $site_name} if "); p.print("$ENV{'SERVER_NAME'} =~ m,^([^.]+(?:\\.[^.]+)?)(?:\\.|$),;\n"); // Assume by default that XSS is a problem, and try to prevent it. // p.print("$prevent_xss = 1;\n"); // Generate URLs using smart http:// // p.print("{\n"); p.print(" my $secure = $ENV{'HTTPS'} =~ /^ON$/i;\n"); p.print(" my $http_url = $secure ? 'https://' : 'http://';\n"); p.print(" $http_url .= qq{$ENV{'GERRIT_USER_NAME'}@}\n"); p.print(" unless $ENV{'GERRIT_ANONYMOUS_READ'};\n"); p.print(" $http_url .= $ENV{'SERVER_NAME'};\n"); p.print(" $http_url .= qq{:$ENV{'SERVER_PORT'}}\n"); p.print(" if (( $secure && $ENV{'SERVER_PORT'} != 443)\n"); p.print(" || (!$secure && $ENV{'SERVER_PORT'} != 80)\n"); p.print(" );\n"); p.print(" $http_url .= qq{$ENV{'GERRIT_CONTEXT_PATH'}p};\n"); p.print(" push @git_base_url_list, $http_url;\n"); p.print("}\n"); // Generate URLs using anonymous git:// // String url = cfg.getString("gerrit", null, "canonicalGitUrl"); if (url != null) { if (url.endsWith("/")) { url = url.substring(0, url.length() - 1); } p.print("if ($ENV{'GERRIT_ANONYMOUS_READ'}) {\n"); p.print(" push @git_base_url_list, "); p.print(quoteForPerl(url)); p.print(";\n"); p.print("}\n"); } // Generate URLs using authenticated ssh:// // if (sshInfo != null && !sshInfo.getHostKeys().isEmpty()) { String sshAddr = sshInfo.getHostKeys().get(0).getHost(); p.print("if ($ENV{'GERRIT_USER_NAME'}) {\n"); p.print(" push @git_base_url_list, join('', 'ssh://'"); p.print(", $ENV{'GERRIT_USER_NAME'}"); p.print(", '@'"); if (sshAddr.startsWith("*:") || "".equals(sshAddr)) { p.print(", $ENV{'SERVER_NAME'}"); } if (sshAddr.startsWith("*")) { sshAddr = sshAddr.substring(1); } p.print(", " + quoteForPerl(sshAddr)); p.print(");\n"); p.print("}\n"); } // Link back to Gerrit (when possible, to matching review record). // Supported Gitweb's hash values are: // - (missing), // - HEAD, // - refs/heads/, // - refs/changes/*//*, // - . // p.print("sub add_review_link {\n"); p.print(" my $h = shift;\n"); p.print(" my $q;\n"); p.print(" if (!$h || $h eq 'HEAD') {\n"); p.print(" $q = qq{#q,project:$ENV{'GERRIT_PROJECT_NAME'}};\n"); p.print(" } elsif ($h =~ /^refs\\/heads\\/([-\\w]+)$/) {\n"); p.print(" $q = qq{#q,project:$ENV{'GERRIT_PROJECT_NAME'}"); // wrapped p.print("+branch:$1};\n"); p.print(" } elsif ($h =~ /^refs\\/changes\\/\\d{2}\\/(\\d+)\\/\\d+$/) "); // wrapped p.print("{\n"); p.print(" $q = qq{#/c/$1};\n"); p.print(" } else {\n"); p.print(" $q = qq{#/q/$h};\n"); p.print(" }\n"); p.print(" my $r = qq{$ENV{'GERRIT_CONTEXT_PATH'}$q};\n"); p.print(" push @{$feature{'actions'}{'default'}},\n"); p.print(" ('review',$r,'commitdiff');\n"); p.print("}\n"); p.print("if ($cgi->param('hb')) {\n"); p.print(" add_review_link($cgi->param('hb'));\n"); p.print("} elsif ($cgi->param('h')) {\n"); p.print(" add_review_link($cgi->param('h'));\n"); p.print("} else {\n"); p.print(" add_review_link();\n"); p.print("}\n"); // If the administrator has created a site-specific gitweb_config, // load that before we perform any final overrides. // Path sitecfg = site.site_gitweb; if (Files.isRegularFile(sitecfg)) { p.print("$GITWEB_CONFIG = " + quoteForPerl(sitecfg) + ";\n"); p.print("if (-e $GITWEB_CONFIG) {\n"); p.print(" do " + quoteForPerl(sitecfg) + ";\n"); p.print("}\n"); } Path root = repoManager.getBasePath(); p.print("$projectroot = " + quoteForPerl(root) + ";\n"); // Permit exporting only the project we were started for. // We use the name under $projectroot in case symlinks // were involved in the path. // p.print("$export_auth_hook = sub {\n"); p.print(" my $dir = shift;\n"); p.print(" my $name = $ENV{'GERRIT_PROJECT_NAME'};\n"); p.print(" my $allow = qq{$projectroot/$name.git};\n"); p.print(" return $dir eq $allow;\n"); p.print(" };\n"); // Do not allow the administrator to enable path info, its // not a URL format we currently support. // p.print("$feature{'pathinfo'}{'override'} = 0;\n"); p.print("$feature{'pathinfo'}{'default'} = [0];\n"); // We don't do forking, so don't allow it to be enabled. // p.print("$feature{'forks'}{'override'} = 0;\n"); p.print("$feature{'forks'}{'default'} = [0];\n"); } myconfFile.setReadOnly();
4790 1242678074GerritCodeReview/gerritDave Borowitza828fedfba116d1f101cae10638c676ce4ff3b22 None TODO: fix setMessage to work without init() SATD_ADDED insertPatchSet(RevCommit) private PatchSet insertPatchSet(RevCommit commit) ProblemInfo p = problem("No patch set found for merged commit " + commit.name()); if (!user.get().isIdentifiedUser()) { p.status = Status.FIX_FAILED; p.outcome = "Must be called by an identified user to insert new patch set"; return null; } try { ChangeControl ctl = changeControlFactory.controlFor(change, user.get()); PatchSetInserter inserter = patchSetInserterFactory.create(repo, rw, ctl, commit); change = inserter.setValidatePolicy(ValidatePolicy.NONE).setRunHooks(false).setSendMail(false).setAllowClosed(true).setUploader(((IdentifiedUser) user.get()).getAccountId()).setMessage("Patch set for merged commit inserted by consistency checker").insert(); p.status = Status.FIXED; p.outcome = "Inserted as patch set " + change.currentPatchSetId().get(); return inserter.getPatchSet(); } catch (InvalidChangeOperationException | OrmException | IOException | NoSuchChangeException e) { warn(e); p.status = Status.FIX_FAILED; p.outcome = "Error inserting new patch set"; return null; }
4789 1242677776GerritCodeReview/gerritDavid Ostrovsky9480fb45b9bcb34f5d88a639628a958d455532ad The identity information reported about the connection by a RFC 1413 [11] request to the remote agent, if available. Servers MAY choose not to support this feature, or not to request the data for efficiency reasons. \"REMOTE_IDENT\" => \"NYI\" The identity information reported about the connection by a RFC 1413 [11] request to the remote agent, if available. Servers MAY choose not to support this feature, or not to request the data for efficiency reasons. \"REMOTE_IDENT\" => \"NYI\" CLASS_OR_METHOD_CHANGED makeSiteConfig(SitePaths, Config, SshInfo) private void makeSiteConfig(SitePaths site, Config cfg, SshInfo sshInfo) throws IOException if (!Files.exists(site.tmp_dir)) { Files.createDirectories(site.tmp_dir); } Path myconf = Files.createTempFile(site.tmp_dir, "gitweb_config", ".perl"); // To make our configuration file only readable or writable by us; // this reduces the chances of someone tampering with the file. // // TODO(dborowitz): Is there a portable way to do this with NIO? File myconfFile = myconf.toFile(); myconfFile.setWritable(false, false); myconfFile.setReadable(false, false); myconfFile.setExecutable(false, false); myconfFile.setWritable(true, true); myconfFile.setReadable(true, true); myconfFile.deleteOnExit(); _env.set("GIT_DIR", "."); _env.set("GITWEB_CONFIG", myconf.toAbsolutePath().toString()); try (PrintWriter p = new PrintWriter(Files.newBufferedWriter(myconf, UTF_8))) { p.print("# Autogenerated by Gerrit Code Review \n"); p.print("# DO NOT EDIT\n"); p.print("\n"); // We are mounted at the same level in the context as the main // UI, so we can include the same header and footer scheme. // Path hdr = site.site_header; if (Files.isRegularFile(hdr)) { p.print("$site_header = " + quoteForPerl(hdr) + ";\n"); } Path ftr = site.site_footer; if (Files.isRegularFile(ftr)) { p.print("$site_footer = " + quoteForPerl(ftr) + ";\n"); } // Top level should return to Gerrit's UI. // p.print("$home_link = $ENV{'GERRIT_CONTEXT_PATH'};\n"); p.print("$home_link_str = 'Code Review';\n"); p.print("$favicon = 'favicon.ico';\n"); p.print("$logo = 'gitweb-logo.png';\n"); p.print("$javascript = 'gitweb.js';\n"); p.print("@stylesheets = ('gitweb-default.css');\n"); Path css = site.site_css; if (Files.isRegularFile(css)) { p.print("push @stylesheets, 'gitweb-site.css';\n"); } // Try to make the title match Gerrit's normal window title // scheme of host followed by 'Code Review'. // p.print("$site_name = $home_link_str;\n"); p.print("$site_name = qq{$1 $site_name} if "); p.print("$ENV{'SERVER_NAME'} =~ m,^([^.]+(?:\\.[^.]+)?)(?:\\.|$),;\n"); // Assume by default that XSS is a problem, and try to prevent it. // p.print("$prevent_xss = 1;\n"); // Generate URLs using smart http:// // p.print("{\n"); p.print(" my $secure = $ENV{'HTTPS'} =~ /^ON$/i;\n"); p.print(" my $http_url = $secure ? 'https://' : 'http://';\n"); p.print(" $http_url .= qq{$ENV{'GERRIT_USER_NAME'}@}\n"); p.print(" unless $ENV{'GERRIT_ANONYMOUS_READ'};\n"); p.print(" $http_url .= $ENV{'SERVER_NAME'};\n"); p.print(" $http_url .= qq{:$ENV{'SERVER_PORT'}}\n"); p.print(" if (( $secure && $ENV{'SERVER_PORT'} != 443)\n"); p.print(" || (!$secure && $ENV{'SERVER_PORT'} != 80)\n"); p.print(" );\n"); p.print(" $http_url .= qq{$ENV{'GERRIT_CONTEXT_PATH'}p};\n"); p.print(" push @git_base_url_list, $http_url;\n"); p.print("}\n"); // Generate URLs using anonymous git:// // String url = cfg.getString("gerrit", null, "canonicalGitUrl"); if (url != null) { if (url.endsWith("/")) { url = url.substring(0, url.length() - 1); } p.print("if ($ENV{'GERRIT_ANONYMOUS_READ'}) {\n"); p.print(" push @git_base_url_list, "); p.print(quoteForPerl(url)); p.print(";\n"); p.print("}\n"); } // Generate URLs using authenticated ssh:// // if (sshInfo != null && !sshInfo.getHostKeys().isEmpty()) { String sshAddr = sshInfo.getHostKeys().get(0).getHost(); p.print("if ($ENV{'GERRIT_USER_NAME'}) {\n"); p.print(" push @git_base_url_list, join('', 'ssh://'"); p.print(", $ENV{'GERRIT_USER_NAME'}"); p.print(", '@'"); if (sshAddr.startsWith("*:") || "".equals(sshAddr)) { p.print(", $ENV{'SERVER_NAME'}"); } if (sshAddr.startsWith("*")) { sshAddr = sshAddr.substring(1); } p.print(", " + quoteForPerl(sshAddr)); p.print(");\n"); p.print("}\n"); } // Link back to Gerrit (when possible, to matching review record). // Supported Gitweb's hash values are: // - (missing), // - HEAD, // - refs/heads/, // - refs/changes/*//*, // - . // p.print("sub add_review_link {\n"); p.print(" my $h = shift;\n"); p.print(" my $q;\n"); p.print(" if (!$h || $h eq 'HEAD') {\n"); p.print(" $q = qq{#q,project:$ENV{'GERRIT_PROJECT_NAME'}};\n"); p.print(" } elsif ($h =~ /^refs\\/heads\\/([-\\w]+)$/) {\n"); p.print(" $q = qq{#q,project:$ENV{'GERRIT_PROJECT_NAME'}"); // wrapped p.print("+branch:$1};\n"); p.print(" } elsif ($h =~ /^refs\\/changes\\/\\d{2}\\/(\\d+)\\/\\d+$/) "); // wrapped p.print("{\n"); p.print(" $q = qq{#/c/$1};\n"); p.print(" } else {\n"); p.print(" $q = qq{#/q/$h};\n"); p.print(" }\n"); p.print(" my $r = qq{$ENV{'GERRIT_CONTEXT_PATH'}$q};\n"); p.print(" push @{$feature{'actions'}{'default'}},\n"); p.print(" ('review',$r,'commitdiff');\n"); p.print("}\n"); p.print("if ($cgi->param('hb')) {\n"); p.print(" add_review_link($cgi->param('hb'));\n"); p.print("} elsif ($cgi->param('h')) {\n"); p.print(" add_review_link($cgi->param('h'));\n"); p.print("} else {\n"); p.print(" add_review_link();\n"); p.print("}\n"); // If the administrator has created a site-specific gitweb_config, // load that before we perform any final overrides. // Path sitecfg = site.site_gitweb; if (Files.isRegularFile(sitecfg)) { p.print("$GITWEB_CONFIG = " + quoteForPerl(sitecfg) + ";\n"); p.print("if (-e $GITWEB_CONFIG) {\n"); p.print(" do " + quoteForPerl(sitecfg) + ";\n"); p.print("}\n"); } Path root = repoManager.getBasePath(); p.print("$projectroot = " + quoteForPerl(root) + ";\n"); // Permit exporting only the project we were started for. // We use the name under $projectroot in case symlinks // were involved in the path. // p.print("$export_auth_hook = sub {\n"); p.print(" my $dir = shift;\n"); p.print(" my $name = $ENV{'GERRIT_PROJECT_NAME'};\n"); p.print(" my $allow = qq{$projectroot/$name.git};\n"); p.print(" return $dir eq $allow;\n"); p.print(" };\n"); // Do not allow the administrator to enable path info, its // not a URL format we currently support. // p.print("$feature{'pathinfo'}{'override'} = 0;\n"); p.print("$feature{'pathinfo'}{'default'} = [0];\n"); // We don't do forking, so don't allow it to be enabled. // p.print("$feature{'forks'}{'override'} = 0;\n"); p.print("$feature{'forks'}{'default'} = [0];\n"); } myconfFile.setReadOnly();
4788 1242678073GerritCodeReview/gerritChristian Aistleitner8d73b76bd911652170578196b2e695b507faea3e None Some object is too large for the merge attempt to succeed. Assume it was a rework. SATD_ADDED module() public static Module module() return new CacheModule() { @Override protected void configure() { bind(ChangeKindCache.class).to(ChangeKindCacheImpl.class); persist(ID_CACHE, Key.class, ChangeKind.class).maximumWeight(2 << 20).weigher(ChangeKindWeigher.class).loader(Loader.class); } };
4787 1242678073GerritCodeReview/gerritDavid Pursehouse75f39623296d6efdffb5adf4b948ed8574651586 None Some object is too large for the merge attempt to succeed. Assume it was a rework. SATD_ADDED module() public static Module module() return new CacheModule() { @Override protected void configure() { bind(ChangeKindCache.class).to(ChangeKindCacheImpl.class); persist(ID_CACHE, Key.class, ChangeKind.class).maximumWeight(2 << 20).weigher(ChangeKindWeigher.class).loader(Loader.class); } };
4785 1242678072GerritCodeReview/gerritStefan Bellerf89a959ad2736280639300ca05af4f452bc179d2 None Not exact, but we cannot do any better. SATD_ADDED ?> topicField(Schema) static FieldDef topicField(Schema schema) if (schema == null) { return ChangeField.LEGACY_TOPIC2; } if (schema.hasField(EXACT_TOPIC)) { return schema.getFields().get(EXACT_TOPIC.getName()); } if (schema.hasField(ChangeField.LEGACY_TOPIC2)) { return schema.getFields().get(ChangeField.LEGACY_TOPIC2.getName()); } // Not exact, but we cannot do any better. return schema.getFields().get(ChangeField.LEGACY_TOPIC3.getName());
4784 1242678071GerritCodeReview/gerritDave Borowitz5cf7109f7ce4fed3659ea93b987bd3cd7693f273 None Expected size of each list is 1, so nested linear search is fine. SATD_ADDED apply(RevisionResource) public RelatedInfo apply(RevisionResource rsrc) throws RepositoryNotFoundException, IOException, OrmException List thisPatchSetGroups = GroupCollector.getGroups(rsrc); if (byAncestorsOnly || thisPatchSetGroups == null || !indexes.getSearchIndex().getSchema().hasField(ChangeField.GROUP)) { return byAncestors.getRelated(rsrc); } RelatedInfo relatedInfo = new RelatedInfo(); relatedInfo.changes = getRelated(rsrc, thisPatchSetGroups); return relatedInfo;
4783 1242678070GerritCodeReview/gerritDave Borowitzc6853e158b9988089f931bdbf8d77985994a68fc None TODO(dborowitz): Tests for octopus merges. SATD_ADDED psId(int, int) private static PatchSet.Id psId(int c, int p) return new PatchSet.Id(new Change.Id(c), p);
4782 1242678069GerritCodeReview/gerritDave Borowitzc6853e158b9988089f931bdbf8d77985994a68fc None Only one parent is new in this push. If it is the only parent, just use that parent's group. If there are multiple parents, perhaps this commit is a merge of a side branch. This commit belongs in that parent's group in that case. SATD_ADDED visit(RevCommit) void visit(RevCommit c) checkState(!done, "visit() called after getGroups()"); Set interestingParents = getInterestingParents(c); if (interestingParents.size() == 0) { // All parents are uninteresting: treat this commit as the root of a new // group of related changes. groups.put(c, c.name()); return; } else if (interestingParents.size() == 1) { // Only one parent is new in this push. If it is the only parent, just use // that parent's group. If there are multiple parents, perhaps this commit // is a merge of a side branch. This commit belongs in that parent's group // in that case. groups.putAll(c, groups.get(interestingParents.iterator().next())); return; } // Multiple parents, merging at least two branches containing new commits in // this push. Set thisCommitGroups = new TreeSet<>(); Set parentGroupsNewInThisPush = Sets.newLinkedHashSetWithExpectedSize(interestingParents.size()); for (RevCommit p : interestingParents) { Collection parentGroups = groups.get(p); if (parentGroups.isEmpty()) { throw new IllegalStateException(String.format("no group assigned to parent %s of commit %s", p.name(), c.name())); } for (String parentGroup : parentGroups) { if (isGroupFromExistingPatchSet(p, parentGroup)) { // This parent's group is from an existing patch set, i.e. the parent // not new in this push. Use this group for the commit. thisCommitGroups.add(parentGroup); } else { // This parent's group is new in this push. parentGroupsNewInThisPush.add(parentGroup); } } } Iterable toAlias; if (thisCommitGroups.isEmpty()) { // All parent groups were new in this push. Pick the first one and alias // other parents' groups to this first parent. String firstParentGroup = parentGroupsNewInThisPush.iterator().next(); thisCommitGroups = ImmutableSet.of(firstParentGroup); toAlias = Iterables.skip(parentGroupsNewInThisPush, 1); } else { // For each parent group that was new in this push, alias it to the actual // computed group(s) for this commit. toAlias = parentGroupsNewInThisPush; } groups.putAll(c, thisCommitGroups); for (String pg : toAlias) { groupAliases.putAll(pg, thisCommitGroups); }
4780 1242678040GerritCodeReview/gerritDave Borowitz4ab21befbce78321b0b1b8e1916996b2c60c708c probably a SHA1 Probably a SHA-1. SATD_CHANGED apply(RevisionResource, RebaseInput) public ChangeInfo apply(RevisionResource rsrc, RebaseInput input) throws AuthException, ResourceNotFoundException, ResourceConflictException, EmailException, OrmException, IOException ChangeControl control = rsrc.getControl(); Change change = rsrc.getChange(); if (!control.canRebase()) { throw new AuthException("rebase not permitted"); } else if (!change.getStatus().isOpen()) { throw new ResourceConflictException("change is " + change.getStatus().name().toLowerCase()); } else if (!hasOneParent(rsrc.getPatchSet().getId())) { throw new ResourceConflictException("cannot rebase merge commits or commit with no ancestor"); } try (Repository repo = repoManager.openRepository(change.getProject()); RevWalk rw = new RevWalk(repo)) { rebaseChange.get().rebase(repo, rw, change, rsrc.getPatchSet().getId(), rsrc.getUser(), findBaseRev(rw, rsrc, input)); } catch (InvalidChangeOperationException e) { throw new ResourceConflictException(e.getMessage()); } catch (NoSuchChangeException e) { throw new ResourceNotFoundException(change.getId().toString()); } return json.format(change.getId());
4779 1242678068GerritCodeReview/gerritDavid Pursehouse7c000e31bbf074a7a29200b7cf30ec2c7a431e29 None If the container didn't do the authentication we might have done it in the front-end web server. Try to split the identity out of the Authorization header and honor it. SATD_ADDED getRemoteUser(HttpServletRequest, String) public static String getRemoteUser(HttpServletRequest req, String loginHeader) if (AUTHORIZATION.equals(loginHeader)) { String user = emptyToNull(req.getRemoteUser()); if (user != null) { // The container performed the authentication, and has the user // identity already decoded for us. Honor that as we have been // configured to honor HTTP authentication. return user; } // If the container didn't do the authentication we might // have done it in the front-end web server. Try to split // the identity out of the Authorization header and honor it. String auth = req.getHeader(AUTHORIZATION); return extractUsername(auth); } else { // Nonstandard HTTP header. We have been told to trust this // header blindly as-is. return emptyToNull(req.getHeader(loginHeader)); }
4777 1242677919GerritCodeReview/gerritEdwin Kempin0ec1e17477a8bb00472837443239eea9a6839133 TODO skip may need to be handled TODO skip may need to be handled CLASS_OR_METHOD_CHANGED text_a() public final String text_a() StringBuilder s = new StringBuilder(); JsArray c = content(); for (int i = 0; i < c.length(); i++) { Region r = c.get(i); if (r.ab() != null) { append(s, r.ab()); } else if (r.a() != null) { append(s, r.a()); } // TODO skip may need to be handled } return s.toString();
4776 1242677918GerritCodeReview/gerritEdwin Kempin0ec1e17477a8bb00472837443239eea9a6839133 TODO skip may need to be handled TODO skip may need to be handled CLASS_OR_METHOD_CHANGED text_a() public final String text_a() StringBuilder s = new StringBuilder(); JsArray c = content(); for (int i = 0; i < c.length(); i++) { Region r = c.get(i); if (r.ab() != null) { append(s, r.ab()); } else if (r.a() != null) { append(s, r.a()); } // TODO skip may need to be handled } return s.toString();
4775 1242678067GerritCodeReview/gerritDavid Ostrovsky5fd7813e72c37ccf14bae374586bf0f5951bbd85 None TODO(dborowitz): Could eliminate this call by adding an option to include inline comments in the change detail. SATD_ADDED loadComments(RevisionInfo, CallbackGroup) private List>> loadComments(final RevisionInfo rev, CallbackGroup group) final List>> r = new ArrayList<>(1); // TODO(dborowitz): Could eliminate this call by adding an option to include // inline comments in the change detail. ChangeApi.comments(changeId.get()).get(group.add(new AsyncCallback>>() { @Override public void onSuccess(NativeMap> result) { // Return value is used for populating the file table, so only count // comments for the current revision. Still include all comments in // the history table. r.add(filterForRevision(result, rev._number())); history.addComments(result); } @Override public void onFailure(Throwable caught) { } })); return r;
4774 1242678066GerritCodeReview/gerritDave Borowitz9d4c51ce49ce773349025ee8e59d2e7df21a2ed2 None TODO(dborowitz): Bypass cache if side is REVISION. SATD_ADDED setCommentRevId(PatchLineComment, PatchListCache, Change, PatchSet) public static RevId setCommentRevId(PatchLineComment c, PatchListCache cache, Change change, PatchSet ps) throws OrmException if (c.getRevId() == null) { try { // TODO(dborowitz): Bypass cache if side is REVISION. PatchList patchList = cache.get(change, ps); c.setRevId((c.getSide() == (short) 0) ? new RevId(ObjectId.toString(patchList.getOldId())) : new RevId(ObjectId.toString(patchList.getNewId()))); } catch (PatchListNotAvailableException e) { throw new OrmException(e); } } return c.getRevId();
4773 1242678065GerritCodeReview/gerritDave Borowitz9d4c51ce49ce773349025ee8e59d2e7df21a2ed2 None TODO(dborowitz): Just scan author space. SATD_ADDED draftByAuthor(ReviewDb, Account.Id) public List draftByAuthor(ReviewDb db, Account.Id author) throws OrmException if (!migration.readChanges()) { return sort(db.patchComments().draftByAuthor(author).toList()); } // TODO(dborowitz): Just scan author space. Set refNames = getRefNamesAllUsers(RefNames.REFS_DRAFT_COMMENTS); List comments = Lists.newArrayList(); for (String refName : refNames) { Account.Id id = Account.Id.fromRefPart(refName); if (!author.equals(id)) { continue; } Change.Id changeId = Change.Id.parse(refName); comments.addAll(draftFactory.create(changeId, author).load().getComments().values()); } return sort(comments);
4772 1242678064GerritCodeReview/gerritDave Borowitz9d4c51ce49ce773349025ee8e59d2e7df21a2ed2 None TODO(dborowitz): Defensive copy? SATD_ADDED PatchLineComment> getComments() public ImmutableListMultimap getComments() // TODO(dborowitz): Defensive copy? return comments;
4771 1242677999GerritCodeReview/gerritDave Borowitz9d4c51ce49ce773349025ee8e59d2e7df21a2ed2 TODO(dborowitz): Use a ThreadLocal or use Joda. TODO(dborowitz): Use a ThreadLocal or use Joda. CLASS_OR_METHOD_CHANGED parseCommentsFromNotes(Repository, String, RevWalk, Change.Id, Multimap, Status) public static NoteMap parseCommentsFromNotes(Repository repo, String refName, RevWalk walk, Change.Id changeId, Multimap comments, Status status) throws IOException, ConfigInvalidException Ref ref = repo.getRef(refName); if (ref == null) { return null; } ObjectReader reader = walk.getObjectReader(); RevCommit commit = walk.parseCommit(ref.getObjectId()); NoteMap noteMap = NoteMap.read(reader, commit); for (Note note : noteMap) { byte[] bytes = reader.open(note.getData(), OBJ_BLOB).getCachedBytes(MAX_NOTE_SZ); List result = parseNote(bytes, changeId, status); if (result == null || result.isEmpty()) { continue; } comments.putAll(new RevId(note.name()), result); } return noteMap;
4770 1242677867GerritCodeReview/gerritDave Borowitzd5ebd9bf010e5801667cdf63ada7f2f3b459c9f6 Missing a trailing LF? Correct it (perhaps the editor was broken). Missing a trailing LF? Correct it (perhaps the editor was broken). CLASS_OR_METHOD_CHANGED createCherryPickCommitMessage(RevCommit, ChangeControl, PatchSet.Id) public String createCherryPickCommitMessage(RevCommit n, ChangeControl ctl, PatchSet.Id psId) Change c = ctl.getChange(); final List footers = n.getFooterLines(); final StringBuilder msgbuf = new StringBuilder(); msgbuf.append(n.getFullMessage()); if (msgbuf.length() == 0) { // WTF, an empty commit message? msgbuf.append(""); } if (msgbuf.charAt(msgbuf.length() - 1) != '\n') { // Missing a trailing LF? Correct it (perhaps the editor was broken). msgbuf.append('\n'); } if (footers.isEmpty()) { // Doesn't end in a "Signed-off-by: ..." style line? Add another line // break to start a new paragraph for the reviewed-by tag lines. // msgbuf.append('\n'); } if (!contains(footers, FooterConstants.CHANGE_ID, c.getKey().get())) { msgbuf.append(FooterConstants.CHANGE_ID.getName()); msgbuf.append(": "); msgbuf.append(c.getKey().get()); msgbuf.append('\n'); } final String siteUrl = urlProvider.get(); if (siteUrl != null) { final String url = siteUrl + c.getId().get(); if (!contains(footers, FooterConstants.REVIEWED_ON, url)) { msgbuf.append(FooterConstants.REVIEWED_ON.getName()); msgbuf.append(": "); msgbuf.append(url); msgbuf.append('\n'); } } PatchSetApproval submitAudit = null; for (final PatchSetApproval a : safeGetApprovals(ctl, psId)) { if (a.getValue() <= 0) { // Negative votes aren't counted. continue; } if (a.isSubmit()) { // Submit is treated specially, below (becomes committer) // if (submitAudit == null || a.getGranted().compareTo(submitAudit.getGranted()) > 0) { submitAudit = a; } continue; } final Account acc = identifiedUserFactory.create(a.getAccountId()).getAccount(); final StringBuilder identbuf = new StringBuilder(); if (acc.getFullName() != null && acc.getFullName().length() > 0) { if (identbuf.length() > 0) { identbuf.append(' '); } identbuf.append(acc.getFullName()); } if (acc.getPreferredEmail() != null && acc.getPreferredEmail().length() > 0) { if (isSignedOffBy(footers, acc.getPreferredEmail())) { continue; } if (identbuf.length() > 0) { identbuf.append(' '); } identbuf.append('<'); identbuf.append(acc.getPreferredEmail()); identbuf.append('>'); } if (identbuf.length() == 0) { // Nothing reasonable to describe them by? Ignore them. continue; } final String tag; if (isCodeReview(a.getLabelId())) { tag = "Reviewed-by"; } else if (isVerified(a.getLabelId())) { tag = "Tested-by"; } else { final LabelType lt = project.getLabelTypes().byLabel(a.getLabelId()); if (lt == null) { continue; } tag = lt.getName(); } if (!contains(footers, new FooterKey(tag), identbuf.toString())) { msgbuf.append(tag); msgbuf.append(": "); msgbuf.append(identbuf); msgbuf.append('\n'); } } return msgbuf.toString();
4764 1242678062GerritCodeReview/gerritDave Borowitzf893c4f87ced3d1c53e7fbf944240c20cfae9d63 None TODO(dborowitz): TEST. SATD_ADDED noteDbEnabled() public static Config noteDbEnabled() return NotesMigration.allEnabledConfig();
4763 1242678061GerritCodeReview/gerritDave Borowitzf893c4f87ced3d1c53e7fbf944240c20cfae9d63 None TODO(dborowitz): Merge this with AcceptanceTestRequestScope. SATD_ADDED module() static Module module() return new AbstractModule() { @Override public void configure() { install(new GerritRequestModule()); bind(RequestScopePropagator.class).to(Propagator.class); bindScope(RequestScoped.class, InProcessProtocol.REQUEST); } @Provides @RemotePeer SocketAddress getSocketAddress() { // TODO(dborowitz): Could potentially fake this with thread ID or // something. throw new OutOfScopeException("No remote peer in acceptance tests"); } };
4762 1242678060GerritCodeReview/gerritDave Borowitzf893c4f87ced3d1c53e7fbf944240c20cfae9d63 None TODO(dborowitz): Could potentially fake this with thread ID or something. SATD_ADDED configure() public void configure() install(new GerritRequestModule()); bind(RequestScopePropagator.class).to(Propagator.class); bindScope(RequestScoped.class, InProcessProtocol.REQUEST);
4761 1242678055GerritCodeReview/gerritDavid Pursehouse107be260546e5c109ba10f98a424dd9472d29b54 no empty paths no suffix no windows/dos style paths no absolute paths no absolute paths no \"l../etc/passwd\" no \"foo/../etc/passwd\" \"foo/./foo\" is insane to ask windows UNC path can be \"//...\" common unix wildcard wildcard or string parameter wildcard Could be used for absolute paths in windows? redirect input redirect output pipe dollar sign carriage return The results of this method are cached by ProjectCacheImpl. Control only enters here if the cache was flushed by the administrator to force scanning the filesystem. Don't rely on the cached names collection. SATD_REMOVED configure() protected void configure() bind(GitRepositoryManager.class).to(LocalDiskRepositoryManager.class); listener().to(LocalDiskRepositoryManager.Lifecycle.class);
4760 1242678055GerritCodeReview/gerritDavid Pursehouse107be260546e5c109ba10f98a424dd9472d29b54 no empty paths no suffix no windows/dos style paths no absolute paths no absolute paths no \"l../etc/passwd\" no \"foo/../etc/passwd\" \"foo/./foo\" is insane to ask windows UNC path can be \"//...\" common unix wildcard wildcard or string parameter wildcard Could be used for absolute paths in windows? redirect input redirect output pipe dollar sign carriage return no empty paths no suffix no windows/dos style paths no absolute paths no absolute paths no \"l../etc/passwd\" no \"foo/../etc/passwd\" \"foo/./foo\" is insane to ask windows UNC path can be \"//...\" no path segments that end with '.git' as \"foo.git/bar\" common unix wildcard wildcard or string parameter wildcard Could be used for absolute paths in windows? redirect input redirect output pipe dollar sign carriage return SATD_CHANGED configure() protected void configure() bind(GitRepositoryManager.class).to(LocalDiskRepositoryManager.class); listener().to(LocalDiskRepositoryManager.Lifecycle.class);
4759 1242678059GerritCodeReview/gerritSaša Živkov92c70260855a77b2e44443920966658b15f98ffa None This is a workaround to be able to print a proper name when the task is wrapped into a ListenableFutureTask. SATD_ADDED start() public void start()
4757 1242678053GerritCodeReview/gerritDavid Pursehouse7273df290a812f28c28e2fbd3dcaccb1e34b4d76 None TODO(dborowitz): Use try/finally when this doesn't double-close the repo. SATD_ADDED doOnlySubscriptionTableOperations(String, Branch.NameKey, List, List) private void doOnlySubscriptionTableOperations(final String gitModulesFileContent, final Branch.NameKey mergedBranch, final List extractedSubscriptions, final List previousSubscriptions) throws Exception expect(schemaFactory.open()).andReturn(schema); try (Repository realDb = createWorkRepository()) { // TODO(dborowitz): Use try/finally when this doesn't double-close the repo. @SuppressWarnings("resource") final Git git = new Git(realDb); addRegularFileToIndex(".gitmodules", gitModulesFileContent, realDb); final RevCommit mergeTip = git.commit().setMessage("test").call(); expect(urlProvider.get()).andReturn("http://localhost:8080").times(2); expect(schema.submoduleSubscriptions()).andReturn(subscriptions); expect(subscriptions.bySuperProject(mergedBranch)).andReturn(new ListResultSet<>(previousSubscriptions)); SortedSet existingProjects = new TreeSet<>(); for (SubmoduleSubscription extracted : extractedSubscriptions) { existingProjects.add(extracted.getSubmodule().getParentKey()); } for (int index = 0; index < extractedSubscriptions.size(); index++) { expect(repoManager.list()).andReturn(existingProjects); } final Set alreadySubscribeds = new HashSet<>(); for (SubmoduleSubscription s : extractedSubscriptions) { if (previousSubscriptions.contains(s)) { alreadySubscribeds.add(s); } } final Set subscriptionsToRemove = new HashSet<>(previousSubscriptions); final List subscriptionsToInsert = new ArrayList<>(extractedSubscriptions); subscriptionsToRemove.removeAll(subscriptionsToInsert); subscriptionsToInsert.removeAll(alreadySubscribeds); if (!subscriptionsToRemove.isEmpty()) { expect(schema.submoduleSubscriptions()).andReturn(subscriptions); subscriptions.delete(subscriptionsToRemove); } expect(schema.submoduleSubscriptions()).andReturn(subscriptions); subscriptions.insert(subscriptionsToInsert); expect(schema.submoduleSubscriptions()).andReturn(subscriptions); expect(subscriptions.bySubmodule(mergedBranch)).andReturn(new ListResultSet<>(new ArrayList())); schema.close(); doReplay(); final SubmoduleOp submoduleOp = new SubmoduleOp(mergedBranch, mergeTip, new RevWalk(realDb), urlProvider, schemaFactory, realDb, new Project(mergedBranch.getParentKey()), new ArrayList(), null, null, repoManager, null, null, null); submoduleOp.update(); }
4756 1242678052GerritCodeReview/gerritDavid Pursehouse7273df290a812f28c28e2fbd3dcaccb1e34b4d76 None TODO(dborowitz): Use try/finally when this doesn't double-close the repo. SATD_ADDED testAvoidingCircularReference() public void testAvoidingCircularReference() throws Exception expect(schemaFactory.open()).andReturn(schema); try (Repository sourceRepository = createWorkRepository(); Repository targetRepository = createWorkRepository()) { // TODO(dborowitz): Use try/finally when this doesn't double-close the repo. @SuppressWarnings("resource") final Git sourceGit = new Git(sourceRepository); // TODO(dborowitz): Use try/finally when this doesn't double-close the repo. @SuppressWarnings("resource") final Git targetGit = new Git(targetRepository); addRegularFileToIndex("file.txt", "test content", sourceRepository); final RevCommit sourceMergeTip = sourceGit.commit().setMessage("test").call(); final Branch.NameKey sourceBranchNameKey = new Branch.NameKey(new Project.NameKey("source-project"), "refs/heads/master"); final CodeReviewCommit codeReviewCommit = new CodeReviewCommit(sourceMergeTip.toObjectId()); final Change submittedChange = new Change(new Change.Key(sourceMergeTip.toObjectId().getName()), new Change.Id(1), new Account.Id(1), sourceBranchNameKey, TimeUtil.nowTs()); final Map mergedCommits = new HashMap<>(); mergedCommits.put(submittedChange.getId(), codeReviewCommit); final List submitted = new ArrayList<>(); submitted.add(submittedChange); addGitLinkToIndex("a", sourceMergeTip.copy(), targetRepository); targetGit.commit().setMessage("test").call(); final Branch.NameKey targetBranchNameKey = new Branch.NameKey(new Project.NameKey("target-project"), sourceBranchNameKey.get()); expect(urlProvider.get()).andReturn("http://localhost:8080"); expect(schema.submoduleSubscriptions()).andReturn(subscriptions); final ResultSet subscribers = new ListResultSet<>(Collections.singletonList(new SubmoduleSubscription(targetBranchNameKey, sourceBranchNameKey, "source-project"))); expect(subscriptions.bySubmodule(sourceBranchNameKey)).andReturn(subscribers); expect(repoManager.openRepository(targetBranchNameKey.getParentKey())).andReturn(targetRepository).anyTimes(); Capture ruCapture = new Capture<>(); gitRefUpdated.fire(eq(targetBranchNameKey.getParentKey()), capture(ruCapture)); changeHooks.doRefUpdatedHook(eq(targetBranchNameKey), anyObject(RefUpdate.class), EasyMock.isNull()); expect(schema.submoduleSubscriptions()).andReturn(subscriptions); final ResultSet incorrectSubscriptions = new ListResultSet<>(Collections.singletonList(new SubmoduleSubscription(sourceBranchNameKey, targetBranchNameKey, "target-project"))); expect(subscriptions.bySubmodule(targetBranchNameKey)).andReturn(incorrectSubscriptions); schema.close(); final PersonIdent myIdent = new PersonIdent("test-user", "test-user@email.com"); doReplay(); final SubmoduleOp submoduleOp = new SubmoduleOp(sourceBranchNameKey, sourceMergeTip, new RevWalk(sourceRepository), urlProvider, schemaFactory, sourceRepository, new Project(sourceBranchNameKey.getParentKey()), submitted, mergedCommits, myIdent, repoManager, gitRefUpdated, null, changeHooks); submoduleOp.update(); doVerify(); RefUpdate ru = ruCapture.getValue(); assertEquals(ru.getName(), targetBranchNameKey.get()); }
4755 1242678051GerritCodeReview/gerritDavid Pursehouse7273df290a812f28c28e2fbd3dcaccb1e34b4d76 None TODO(dborowitz): Use try/finally when this doesn't double-close the repo. SATD_ADDED testOneSubscriberToUpdate() public void testOneSubscriberToUpdate() throws Exception expect(schemaFactory.open()).andReturn(schema); try (Repository sourceRepository = createWorkRepository(); Repository targetRepository = createWorkRepository()) { // TODO(dborowitz): Use try/finally when this doesn't double-close the repo. @SuppressWarnings("resource") final Git sourceGit = new Git(sourceRepository); // TODO(dborowitz): Use try/finally when this doesn't double-close the repo. @SuppressWarnings("resource") final Git targetGit = new Git(targetRepository); addRegularFileToIndex("file.txt", "test content", sourceRepository); final RevCommit sourceMergeTip = sourceGit.commit().setMessage("test").call(); final Branch.NameKey sourceBranchNameKey = new Branch.NameKey(new Project.NameKey("source-project"), "refs/heads/master"); final CodeReviewCommit codeReviewCommit = new CodeReviewCommit(sourceMergeTip.toObjectId()); final Change submittedChange = new Change(new Change.Key(sourceMergeTip.toObjectId().getName()), new Change.Id(1), new Account.Id(1), sourceBranchNameKey, TimeUtil.nowTs()); final Map mergedCommits = new HashMap<>(); mergedCommits.put(submittedChange.getId(), codeReviewCommit); final List submitted = new ArrayList<>(); submitted.add(submittedChange); addGitLinkToIndex("a", sourceMergeTip.copy(), targetRepository); targetGit.commit().setMessage("test").call(); final Branch.NameKey targetBranchNameKey = new Branch.NameKey(new Project.NameKey("target-project"), sourceBranchNameKey.get()); expect(urlProvider.get()).andReturn("http://localhost:8080"); expect(schema.submoduleSubscriptions()).andReturn(subscriptions); final ResultSet subscribers = new ListResultSet<>(Collections.singletonList(new SubmoduleSubscription(targetBranchNameKey, sourceBranchNameKey, "source-project"))); expect(subscriptions.bySubmodule(sourceBranchNameKey)).andReturn(subscribers); expect(repoManager.openRepository(targetBranchNameKey.getParentKey())).andReturn(targetRepository).anyTimes(); Capture ruCapture = new Capture<>(); gitRefUpdated.fire(eq(targetBranchNameKey.getParentKey()), capture(ruCapture)); changeHooks.doRefUpdatedHook(eq(targetBranchNameKey), anyObject(RefUpdate.class), EasyMock.isNull()); expect(schema.submoduleSubscriptions()).andReturn(subscriptions); final ResultSet emptySubscriptions = new ListResultSet<>(new ArrayList()); expect(subscriptions.bySubmodule(targetBranchNameKey)).andReturn(emptySubscriptions); schema.close(); final PersonIdent myIdent = new PersonIdent("test-user", "test-user@email.com"); doReplay(); final SubmoduleOp submoduleOp = new SubmoduleOp(sourceBranchNameKey, sourceMergeTip, new RevWalk(sourceRepository), urlProvider, schemaFactory, sourceRepository, new Project(sourceBranchNameKey.getParentKey()), submitted, mergedCommits, myIdent, repoManager, gitRefUpdated, null, changeHooks); submoduleOp.update(); doVerify(); RefUpdate ru = ruCapture.getValue(); assertEquals(ru.getName(), targetBranchNameKey.get()); }
4754 1242678050GerritCodeReview/gerritDavid Pursehouse7273df290a812f28c28e2fbd3dcaccb1e34b4d76 None TODO(dborowitz): Use try/finally when this doesn't double-close the repo. SATD_ADDED testEmptyCommit() public void testEmptyCommit() throws Exception expect(schemaFactory.open()).andReturn(schema); try (Repository realDb = createWorkRepository()) { // TODO(dborowitz): Use try/finally when this doesn't double-close the repo. @SuppressWarnings("resource") final Git git = new Git(realDb); final RevCommit mergeTip = git.commit().setMessage("test").call(); final Branch.NameKey branchNameKey = new Branch.NameKey(new Project.NameKey("test-project"), "test-branch"); expect(urlProvider.get()).andReturn("http://localhost:8080"); expect(schema.submoduleSubscriptions()).andReturn(subscriptions); final ResultSet emptySubscriptions = new ListResultSet<>(new ArrayList()); expect(subscriptions.bySubmodule(branchNameKey)).andReturn(emptySubscriptions); schema.close(); doReplay(); final SubmoduleOp submoduleOp = new SubmoduleOp(branchNameKey, mergeTip, new RevWalk(realDb), urlProvider, schemaFactory, realDb, null, new ArrayList(), null, null, null, null, null, null); submoduleOp.update(); doVerify(); }
4753 1242678049GerritCodeReview/gerritDavid Pursehouse7273df290a812f28c28e2fbd3dcaccb1e34b4d76 None TODO(dborowitz): Use try/finally when this doesn't double-close the repo. SATD_ADDED setUp() public void setUp() throws Exception super.setUp(); /*- The following graph will be created. o tag 2.5, 2.5_annotated, 2.5_annotated_twice |\ | o tag 2.0.1 | o tag 2.0 o | tag 1.3 |/ o c3 | o tag 1.0.1 |/ o tag 1.0 o c2 o c1 */ // TODO(dborowitz): Use try/finally when this doesn't double-close the repo. @SuppressWarnings("resource") Git git = new Git(db); revWalk = new RevWalk(db); // Version 1.0 commit_initial = git.commit().setMessage("c1").call(); git.commit().setMessage("c2").call(); RevCommit commit_v1_0 = git.commit().setMessage("version 1.0").call(); git.tag().setName(TAG_1_0).setObjectId(commit_v1_0).call(); RevCommit c3 = git.commit().setMessage("c3").call(); // Version 1.01 createAndCheckoutBranch(commit_v1_0, BRANCH_1_0); RevCommit commit_v1_0_1 = git.commit().setMessage("verREFS_HEADS_RELsion 1.0.1").call(); git.tag().setName(TAG_1_0_1).setObjectId(commit_v1_0_1).call(); // Version 1.3 createAndCheckoutBranch(c3, BRANCH_1_3); commit_v1_3 = git.commit().setMessage("version 1.3").call(); git.tag().setName(TAG_1_3).setObjectId(commit_v1_3).call(); // Version 2.0 createAndCheckoutBranch(c3, BRANCH_2_0); RevCommit commit_v2_0 = git.commit().setMessage("version 2.0").call(); git.tag().setName(TAG_2_0).setObjectId(commit_v2_0).call(); RevCommit commit_v2_0_1 = git.commit().setMessage("version 2.0.1").call(); git.tag().setName(TAG_2_0_1).setObjectId(commit_v2_0_1).call(); // Version 2.5 createAndCheckoutBranch(commit_v1_3, BRANCH_2_5); git.merge().include(commit_v2_0_1).setCommit(false).setFastForward(FastForwardMode.NO_FF).call(); commit_v2_5 = git.commit().setMessage("version 2.5").call(); git.tag().setName(TAG_2_5).setObjectId(commit_v2_5).setAnnotated(false).call(); Ref ref_tag_2_5_annotated = git.tag().setName(TAG_2_5_ANNOTATED).setObjectId(commit_v2_5).setAnnotated(true).call(); RevTag tag_2_5_annotated = revWalk.parseTag(ref_tag_2_5_annotated.getObjectId()); git.tag().setName(TAG_2_5_ANNOTATED_TWICE).setObjectId(tag_2_5_annotated).setAnnotated(true).call();
4752 1242678058GerritCodeReview/gerritDave Borowitz70aaf7eaa75775004498bc9e931c79b712486705 None TODO(dborowitz): Delete when we delete support for pre-Lucene-5.0 schemas. SATD_ADDED setReady(SitePaths, int, boolean) public static void setReady(SitePaths sitePaths, int version, boolean ready) throws IOException try { FileBasedConfig cfg = LuceneVersionManager.loadGerritIndexConfig(sitePaths); LuceneVersionManager.setReady(cfg, version, ready); cfg.save(); } catch (ConfigInvalidException e) { throw new IOException(e); }
4751 1242678057GerritCodeReview/gerritDavid Pursehouse304130b0674adad07e5c9576bd366b5b1021e1b6 None TODO: remove this code when Guice fixes its issue 745 SATD_ADDED configure() protected void configure() bind(Resolver.class); bind(UploadFactory.class); bind(UploadFilter.class); bind(new TypeLiteral>() { }).to(enableReceive ? ReceiveFactory.class : DisabledReceiveFactory.class); bind(ReceiveFilter.class); install(new CacheModule() { @Override protected void configure() { cache(ID_CACHE, AdvertisedObjectsCacheKey.class, new TypeLiteral>() { }).maximumWeight(4096).expireAfterWrite(10, TimeUnit.MINUTES); } });
4750 1242677875GerritCodeReview/gerritDavid Pursehouse88bcb4857926b27a6c922fb214b38446571ed282 TODO: Implement delete of dashboards by API. TODO: Implement delete of dashboards by API. CLASS_OR_METHOD_CHANGED apply(DashboardResource, SetDashboard.Input) public Response apply(DashboardResource resource, SetDashboard.Input input) throws AuthException, BadRequestException, ResourceConflictException, ResourceNotFoundException, MethodNotAllowedException, IOException if (resource.isProjectDefault()) { SetDashboard.Input in = new SetDashboard.Input(); in.commitMessage = input != null ? input.commitMessage : null; return defaultSetter.get().apply(resource, in); } // TODO: Implement delete of dashboards by API. throw new MethodNotAllowedException();
4749 1242678056GerritCodeReview/gerritDavid Pursehouse1abfd35d76cf5147dd6706214a2e7b3fdc7a5d74 no \"l../etc/passwd\" no \"foo/../etc/passwd\" \"foo/./foo\" is insane to ask windows UNC path can be \"//...\" common unix wildcard wildcard or string parameter wildcard Could be used for absolute paths in windows? redirect input redirect output pipe dollar sign carriage return The results of this method are cached by ProjectCacheImpl. Control only enters here if the cache was flushed by the administrator to force scanning the filesystem. Don't rely on the cached names collection. SATD_REMOVED configure() protected void configure() bind(GitRepositoryManager.class).to(LocalDiskRepositoryManager.class); listener().to(LocalDiskRepositoryManager.Lifecycle.class);
4748 1242678055GerritCodeReview/gerritDavid Pursehouse1abfd35d76cf5147dd6706214a2e7b3fdc7a5d74 no \"l../etc/passwd\" no \"foo/../etc/passwd\" \"foo/./foo\" is insane to ask windows UNC path can be \"//...\" common unix wildcard wildcard or string parameter wildcard Could be used for absolute paths in windows? redirect input redirect output pipe dollar sign carriage return no empty paths no suffix no windows/dos style paths no absolute paths no absolute paths no \"l../etc/passwd\" no \"foo/../etc/passwd\" \"foo/./foo\" is insane to ask windows UNC path can be \"//...\" common unix wildcard wildcard or string parameter wildcard Could be used for absolute paths in windows? redirect input redirect output pipe dollar sign carriage return SATD_CHANGED configure() protected void configure() bind(GitRepositoryManager.class).to(LocalDiskRepositoryManager.class); listener().to(LocalDiskRepositoryManager.Lifecycle.class);
4747 1242677837GerritCodeReview/gerritSaša Živkova4e0250cbdf9344035bcf9168b59e8e3d52ddbb1 For a file with content merge conflict that we produced a result above on, collapse the file down to a single stage 0 with just the blob content, and a randomly selected mode (the lowest stage, which should be the merge base, or ours). For a file with content merge conflict that we produced a result above on, collapse the file down to a single stage 0 with just the blob content, and a randomly selected mode (the lowest stage, which should be the merge base, or ours). CLASS_OR_METHOD_CHANGED automerge(Repository, RevWalk, RevCommit, ThreeWayMergeStrategy, boolean) public static RevTree automerge(Repository repo, RevWalk rw, RevCommit b, ThreeWayMergeStrategy mergeStrategy, boolean save) throws IOException String hash = b.name(); String refName = RefNames.REFS_CACHE_AUTOMERGE + hash.substring(0, 2) + "/" + hash.substring(2); Ref ref = repo.getRef(refName); if (ref != null && ref.getObjectId() != null) { return rw.parseTree(ref.getObjectId()); } ResolveMerger m = (ResolveMerger) mergeStrategy.newMerger(repo, true); final ObjectInserter ins = repo.newObjectInserter(); try { DirCache dc = DirCache.newInCore(); m.setDirCache(dc); m.setObjectInserter(new ObjectInserter.Filter() { @Override protected ObjectInserter delegate() { return ins; } @Override public void flush() { } @Override public void release() { } }); boolean couldMerge; try { couldMerge = m.merge(b.getParents()); } catch (IOException e) { // It is not safe to continue further down in this method as throwing // an exception most likely means that the merge tree was not created // and m.getMergeResults() is empty. This would mean that all paths are // unmerged and Gerrit UI would show all paths in the patch list. log.warn("Error attempting automerge " + refName, e); return null; } ObjectId treeId; if (couldMerge) { treeId = m.getResultTreeId(); } else { RevCommit ours = b.getParent(0); RevCommit theirs = b.getParent(1); rw.parseBody(ours); rw.parseBody(theirs); String oursMsg = ours.getShortMessage(); String theirsMsg = theirs.getShortMessage(); String oursName = String.format("HEAD (%s %s)", ours.abbreviate(6).name(), oursMsg.substring(0, Math.min(oursMsg.length(), 60))); String theirsName = String.format("BRANCH (%s %s)", theirs.abbreviate(6).name(), theirsMsg.substring(0, Math.min(theirsMsg.length(), 60))); MergeFormatter fmt = new MergeFormatter(); Map> r = m.getMergeResults(); Map resolved = new HashMap<>(); for (Map.Entry> entry : r.entrySet()) { MergeResult p = entry.getValue(); TemporaryBuffer buf = new TemporaryBuffer.LocalFile(10 * 1024 * 1024); try { fmt.formatMerge(buf, p, "BASE", oursName, theirsName, "UTF-8"); buf.close(); InputStream in = buf.openInputStream(); try { resolved.put(entry.getKey(), ins.insert(Constants.OBJ_BLOB, buf.length(), in)); } finally { in.close(); } } finally { buf.destroy(); } } DirCacheBuilder builder = dc.builder(); int cnt = dc.getEntryCount(); for (int i = 0; i < cnt; ) { DirCacheEntry entry = dc.getEntry(i); if (entry.getStage() == 0) { builder.add(entry); i++; continue; } int next = dc.nextEntry(i); String path = entry.getPathString(); DirCacheEntry res = new DirCacheEntry(path); if (resolved.containsKey(path)) { // For a file with content merge conflict that we produced a result // above on, collapse the file down to a single stage 0 with just // the blob content, and a randomly selected mode (the lowest stage, // which should be the merge base, or ours). res.setFileMode(entry.getFileMode()); res.setObjectId(resolved.get(path)); } else if (next == i + 1) { // If there is exactly one stage present, shouldn't be a conflict... res.setFileMode(entry.getFileMode()); res.setObjectId(entry.getObjectId()); } else if (next == i + 2) { // Two stages suggests a delete/modify conflict. Pick the higher // stage as the automatic result. entry = dc.getEntry(i + 1); res.setFileMode(entry.getFileMode()); res.setObjectId(entry.getObjectId()); } else { // 3 stage conflict, no resolve above // Punt on the 3-stage conflict and show the base, for now. res.setFileMode(entry.getFileMode()); res.setObjectId(entry.getObjectId()); } builder.add(res); i = next; } builder.finish(); treeId = dc.writeTree(ins); } ins.flush(); if (save) { RefUpdate update = repo.updateRef(refName); update.setNewObjectId(treeId); update.disableRefLog(); update.forceUpdate(); } return rw.lookupTree(treeId); } finally { ins.release(); }
4746 1242678048GerritCodeReview/gerritShawn Pearcecc349904933c20509e390b56d3b04b9a6fc51a1d TODO: fixify this loop TODO: fixify this loop FILE_PATH_CHANGED createJar(Path, List, File, ObjectId, ObjectId) private void createJar(Path archiveFile, List toBeJared, File tempDir, ObjectId metaConfig, ObjectId rulesId) throws IOException long now = TimeUtil.nowMs(); Manifest mf = new Manifest(); mf.getMainAttributes().put(Attributes.Name.MANIFEST_VERSION, "1.0"); mf.getMainAttributes().putValue("Built-by", "Gerrit Code Review " + Version.getVersion()); if (git.getDirectory() != null) { mf.getMainAttributes().putValue("Source-Repository", git.getDirectory().getPath()); } mf.getMainAttributes().putValue("Source-Commit", metaConfig.name()); mf.getMainAttributes().putValue("Source-Blob", rulesId.name()); Path tmpjar = Files.createTempFile(archiveFile.getParent(), ".rulec_", ".jar"); try (OutputStream stream = Files.newOutputStream(tmpjar); JarOutputStream out = new JarOutputStream(stream, mf)) { byte[] buffer = new byte[10240]; // TODO: fixify this loop for (String path : toBeJared) { JarEntry jarAdd = new JarEntry(path); File f = new File(tempDir, path); jarAdd.setTime(now); out.putNextEntry(jarAdd); if (f.isFile()) { FileInputStream in = new FileInputStream(f); try { while (true) { int nRead = in.read(buffer, 0, buffer.length); if (nRead <= 0) { break; } out.write(buffer, 0, nRead); } } finally { in.close(); } } out.closeEntry(); } } try { Files.move(tmpjar, archiveFile); } catch (IOException e) { throw new IOException("Cannot replace " + archiveFile, e); }
4745 1242678054GerritCodeReview/gerritMarco Miller4ff91616a1fa2c79cef3b5e78b3f057edcc2c7de None TODO(dborowitz): Use different project names. SATD_ADDED apply(Statement, Description) public Statement apply(final Statement base, final Description description) return new Statement() { @Override public void evaluate() throws Throwable { boolean mem = description.getAnnotation(UseLocalDisk.class) == null; boolean enableHttpd = description.getAnnotation(NoHttpd.class) == null && description.getTestClass().getAnnotation(NoHttpd.class) == null; beforeTest(description, mem, enableHttpd); base.evaluate(); afterTest(); } };
4744 1242678053GerritCodeReview/gerritDave Borowitz16f552165aac8497576a5fded23d7027e951c547 None TODO(dborowitz): Use try/finally when this doesn't double-close the repo. SATD_ADDED doOnlySubscriptionTableOperations(String, Branch.NameKey, List, List) private void doOnlySubscriptionTableOperations(final String gitModulesFileContent, final Branch.NameKey mergedBranch, final List extractedSubscriptions, final List previousSubscriptions) throws Exception expect(schemaFactory.open()).andReturn(schema); try (Repository realDb = createWorkRepository()) { // TODO(dborowitz): Use try/finally when this doesn't double-close the repo. @SuppressWarnings("resource") final Git git = new Git(realDb); addRegularFileToIndex(".gitmodules", gitModulesFileContent, realDb); final RevCommit mergeTip = git.commit().setMessage("test").call(); expect(urlProvider.get()).andReturn("http://localhost:8080").times(2); expect(schema.submoduleSubscriptions()).andReturn(subscriptions); expect(subscriptions.bySuperProject(mergedBranch)).andReturn(new ListResultSet<>(previousSubscriptions)); SortedSet existingProjects = new TreeSet<>(); for (SubmoduleSubscription extracted : extractedSubscriptions) { existingProjects.add(extracted.getSubmodule().getParentKey()); } for (int index = 0; index < extractedSubscriptions.size(); index++) { expect(repoManager.list()).andReturn(existingProjects); } final Set alreadySubscribeds = new HashSet<>(); for (SubmoduleSubscription s : extractedSubscriptions) { if (previousSubscriptions.contains(s)) { alreadySubscribeds.add(s); } } final Set subscriptionsToRemove = new HashSet<>(previousSubscriptions); final List subscriptionsToInsert = new ArrayList<>(extractedSubscriptions); subscriptionsToRemove.removeAll(subscriptionsToInsert); subscriptionsToInsert.removeAll(alreadySubscribeds); if (!subscriptionsToRemove.isEmpty()) { expect(schema.submoduleSubscriptions()).andReturn(subscriptions); subscriptions.delete(subscriptionsToRemove); } expect(schema.submoduleSubscriptions()).andReturn(subscriptions); subscriptions.insert(subscriptionsToInsert); expect(schema.submoduleSubscriptions()).andReturn(subscriptions); expect(subscriptions.bySubmodule(mergedBranch)).andReturn(new ListResultSet<>(new ArrayList())); schema.close(); doReplay(); final SubmoduleOp submoduleOp = new SubmoduleOp(mergedBranch, mergeTip, new RevWalk(realDb), urlProvider, schemaFactory, realDb, new Project(mergedBranch.getParentKey()), new ArrayList(), null, null, repoManager, null, null, null); submoduleOp.update(); }
4743 1242678052GerritCodeReview/gerritDave Borowitz16f552165aac8497576a5fded23d7027e951c547 None TODO(dborowitz): Use try/finally when this doesn't double-close the repo. SATD_ADDED testAvoidingCircularReference() public void testAvoidingCircularReference() throws Exception expect(schemaFactory.open()).andReturn(schema); try (Repository sourceRepository = createWorkRepository(); Repository targetRepository = createWorkRepository()) { // TODO(dborowitz): Use try/finally when this doesn't double-close the repo. @SuppressWarnings("resource") final Git sourceGit = new Git(sourceRepository); // TODO(dborowitz): Use try/finally when this doesn't double-close the repo. @SuppressWarnings("resource") final Git targetGit = new Git(targetRepository); addRegularFileToIndex("file.txt", "test content", sourceRepository); final RevCommit sourceMergeTip = sourceGit.commit().setMessage("test").call(); final Branch.NameKey sourceBranchNameKey = new Branch.NameKey(new Project.NameKey("source-project"), "refs/heads/master"); final CodeReviewCommit codeReviewCommit = new CodeReviewCommit(sourceMergeTip.toObjectId()); final Change submittedChange = new Change(new Change.Key(sourceMergeTip.toObjectId().getName()), new Change.Id(1), new Account.Id(1), sourceBranchNameKey, TimeUtil.nowTs()); final Map mergedCommits = new HashMap<>(); mergedCommits.put(submittedChange.getId(), codeReviewCommit); final List submitted = new ArrayList<>(); submitted.add(submittedChange); addGitLinkToIndex("a", sourceMergeTip.copy(), targetRepository); targetGit.commit().setMessage("test").call(); final Branch.NameKey targetBranchNameKey = new Branch.NameKey(new Project.NameKey("target-project"), sourceBranchNameKey.get()); expect(urlProvider.get()).andReturn("http://localhost:8080"); expect(schema.submoduleSubscriptions()).andReturn(subscriptions); final ResultSet subscribers = new ListResultSet<>(Collections.singletonList(new SubmoduleSubscription(targetBranchNameKey, sourceBranchNameKey, "source-project"))); expect(subscriptions.bySubmodule(sourceBranchNameKey)).andReturn(subscribers); expect(repoManager.openRepository(targetBranchNameKey.getParentKey())).andReturn(targetRepository).anyTimes(); Capture ruCapture = new Capture<>(); gitRefUpdated.fire(eq(targetBranchNameKey.getParentKey()), capture(ruCapture)); changeHooks.doRefUpdatedHook(eq(targetBranchNameKey), anyObject(RefUpdate.class), EasyMock.isNull()); expect(schema.submoduleSubscriptions()).andReturn(subscriptions); final ResultSet incorrectSubscriptions = new ListResultSet<>(Collections.singletonList(new SubmoduleSubscription(sourceBranchNameKey, targetBranchNameKey, "target-project"))); expect(subscriptions.bySubmodule(targetBranchNameKey)).andReturn(incorrectSubscriptions); schema.close(); final PersonIdent myIdent = new PersonIdent("test-user", "test-user@email.com"); doReplay(); final SubmoduleOp submoduleOp = new SubmoduleOp(sourceBranchNameKey, sourceMergeTip, new RevWalk(sourceRepository), urlProvider, schemaFactory, sourceRepository, new Project(sourceBranchNameKey.getParentKey()), submitted, mergedCommits, myIdent, repoManager, gitRefUpdated, null, changeHooks); submoduleOp.update(); doVerify(); RefUpdate ru = ruCapture.getValue(); assertEquals(ru.getName(), targetBranchNameKey.get()); }
4742 1242678051GerritCodeReview/gerritDave Borowitz16f552165aac8497576a5fded23d7027e951c547 None TODO(dborowitz): Use try/finally when this doesn't double-close the repo. SATD_ADDED testOneSubscriberToUpdate() public void testOneSubscriberToUpdate() throws Exception expect(schemaFactory.open()).andReturn(schema); try (Repository sourceRepository = createWorkRepository(); Repository targetRepository = createWorkRepository()) { // TODO(dborowitz): Use try/finally when this doesn't double-close the repo. @SuppressWarnings("resource") final Git sourceGit = new Git(sourceRepository); // TODO(dborowitz): Use try/finally when this doesn't double-close the repo. @SuppressWarnings("resource") final Git targetGit = new Git(targetRepository); addRegularFileToIndex("file.txt", "test content", sourceRepository); final RevCommit sourceMergeTip = sourceGit.commit().setMessage("test").call(); final Branch.NameKey sourceBranchNameKey = new Branch.NameKey(new Project.NameKey("source-project"), "refs/heads/master"); final CodeReviewCommit codeReviewCommit = new CodeReviewCommit(sourceMergeTip.toObjectId()); final Change submittedChange = new Change(new Change.Key(sourceMergeTip.toObjectId().getName()), new Change.Id(1), new Account.Id(1), sourceBranchNameKey, TimeUtil.nowTs()); final Map mergedCommits = new HashMap<>(); mergedCommits.put(submittedChange.getId(), codeReviewCommit); final List submitted = new ArrayList<>(); submitted.add(submittedChange); addGitLinkToIndex("a", sourceMergeTip.copy(), targetRepository); targetGit.commit().setMessage("test").call(); final Branch.NameKey targetBranchNameKey = new Branch.NameKey(new Project.NameKey("target-project"), sourceBranchNameKey.get()); expect(urlProvider.get()).andReturn("http://localhost:8080"); expect(schema.submoduleSubscriptions()).andReturn(subscriptions); final ResultSet subscribers = new ListResultSet<>(Collections.singletonList(new SubmoduleSubscription(targetBranchNameKey, sourceBranchNameKey, "source-project"))); expect(subscriptions.bySubmodule(sourceBranchNameKey)).andReturn(subscribers); expect(repoManager.openRepository(targetBranchNameKey.getParentKey())).andReturn(targetRepository).anyTimes(); Capture ruCapture = new Capture<>(); gitRefUpdated.fire(eq(targetBranchNameKey.getParentKey()), capture(ruCapture)); changeHooks.doRefUpdatedHook(eq(targetBranchNameKey), anyObject(RefUpdate.class), EasyMock.isNull()); expect(schema.submoduleSubscriptions()).andReturn(subscriptions); final ResultSet emptySubscriptions = new ListResultSet<>(new ArrayList()); expect(subscriptions.bySubmodule(targetBranchNameKey)).andReturn(emptySubscriptions); schema.close(); final PersonIdent myIdent = new PersonIdent("test-user", "test-user@email.com"); doReplay(); final SubmoduleOp submoduleOp = new SubmoduleOp(sourceBranchNameKey, sourceMergeTip, new RevWalk(sourceRepository), urlProvider, schemaFactory, sourceRepository, new Project(sourceBranchNameKey.getParentKey()), submitted, mergedCommits, myIdent, repoManager, gitRefUpdated, null, changeHooks); submoduleOp.update(); doVerify(); RefUpdate ru = ruCapture.getValue(); assertEquals(ru.getName(), targetBranchNameKey.get()); }
4741 1242678050GerritCodeReview/gerritDave Borowitz16f552165aac8497576a5fded23d7027e951c547 None TODO(dborowitz): Use try/finally when this doesn't double-close the repo. SATD_ADDED testEmptyCommit() public void testEmptyCommit() throws Exception expect(schemaFactory.open()).andReturn(schema); try (Repository realDb = createWorkRepository()) { // TODO(dborowitz): Use try/finally when this doesn't double-close the repo. @SuppressWarnings("resource") final Git git = new Git(realDb); final RevCommit mergeTip = git.commit().setMessage("test").call(); final Branch.NameKey branchNameKey = new Branch.NameKey(new Project.NameKey("test-project"), "test-branch"); expect(urlProvider.get()).andReturn("http://localhost:8080"); expect(schema.submoduleSubscriptions()).andReturn(subscriptions); final ResultSet emptySubscriptions = new ListResultSet<>(new ArrayList()); expect(subscriptions.bySubmodule(branchNameKey)).andReturn(emptySubscriptions); schema.close(); doReplay(); final SubmoduleOp submoduleOp = new SubmoduleOp(branchNameKey, mergeTip, new RevWalk(realDb), urlProvider, schemaFactory, realDb, null, new ArrayList(), null, null, null, null, null, null); submoduleOp.update(); doVerify(); }
4740 1242678049GerritCodeReview/gerritDave Borowitz16f552165aac8497576a5fded23d7027e951c547 None TODO(dborowitz): Use try/finally when this doesn't double-close the repo. SATD_ADDED setUp() public void setUp() throws Exception super.setUp(); /*- The following graph will be created. o tag 2.5, 2.5_annotated, 2.5_annotated_twice |\ | o tag 2.0.1 | o tag 2.0 o | tag 1.3 |/ o c3 | o tag 1.0.1 |/ o tag 1.0 o c2 o c1 */ // TODO(dborowitz): Use try/finally when this doesn't double-close the repo. @SuppressWarnings("resource") Git git = new Git(db); revWalk = new RevWalk(db); // Version 1.0 commit_initial = git.commit().setMessage("c1").call(); git.commit().setMessage("c2").call(); RevCommit commit_v1_0 = git.commit().setMessage("version 1.0").call(); git.tag().setName(TAG_1_0).setObjectId(commit_v1_0).call(); RevCommit c3 = git.commit().setMessage("c3").call(); // Version 1.01 createAndCheckoutBranch(commit_v1_0, BRANCH_1_0); RevCommit commit_v1_0_1 = git.commit().setMessage("verREFS_HEADS_RELsion 1.0.1").call(); git.tag().setName(TAG_1_0_1).setObjectId(commit_v1_0_1).call(); // Version 1.3 createAndCheckoutBranch(c3, BRANCH_1_3); commit_v1_3 = git.commit().setMessage("version 1.3").call(); git.tag().setName(TAG_1_3).setObjectId(commit_v1_3).call(); // Version 2.0 createAndCheckoutBranch(c3, BRANCH_2_0); RevCommit commit_v2_0 = git.commit().setMessage("version 2.0").call(); git.tag().setName(TAG_2_0).setObjectId(commit_v2_0).call(); RevCommit commit_v2_0_1 = git.commit().setMessage("version 2.0.1").call(); git.tag().setName(TAG_2_0_1).setObjectId(commit_v2_0_1).call(); // Version 2.5 createAndCheckoutBranch(commit_v1_3, BRANCH_2_5); git.merge().include(commit_v2_0_1).setCommit(false).setFastForward(FastForwardMode.NO_FF).call(); commit_v2_5 = git.commit().setMessage("version 2.5").call(); git.tag().setName(TAG_2_5).setObjectId(commit_v2_5).setAnnotated(false).call(); Ref ref_tag_2_5_annotated = git.tag().setName(TAG_2_5_ANNOTATED).setObjectId(commit_v2_5).setAnnotated(true).call(); RevTag tag_2_5_annotated = revWalk.parseTag(ref_tag_2_5_annotated.getObjectId()); git.tag().setName(TAG_2_5_ANNOTATED_TWICE).setObjectId(tag_2_5_annotated).setAnnotated(true).call();
4739 1242678048GerritCodeReview/gerritDave Borowitz62e67bede79a94381142054a6335a66890595a88 None TODO: fixify this loop SATD_ADDED createJar(Path, List, File, ObjectId, ObjectId) private void createJar(Path archiveFile, List toBeJared, File tempDir, ObjectId metaConfig, ObjectId rulesId) throws IOException long now = TimeUtil.nowMs(); Manifest mf = new Manifest(); mf.getMainAttributes().put(Attributes.Name.MANIFEST_VERSION, "1.0"); mf.getMainAttributes().putValue("Built-by", "Gerrit Code Review " + Version.getVersion()); if (git.getDirectory() != null) { mf.getMainAttributes().putValue("Source-Repository", git.getDirectory().getPath()); } mf.getMainAttributes().putValue("Source-Commit", metaConfig.name()); mf.getMainAttributes().putValue("Source-Blob", rulesId.name()); Path tmpjar = Files.createTempFile(archiveFile.getParent(), ".rulec_", ".jar"); try (OutputStream stream = Files.newOutputStream(tmpjar, DELETE_ON_CLOSE); JarOutputStream out = new JarOutputStream(stream, mf)) { byte[] buffer = new byte[10240]; // TODO: fixify this loop for (String path : toBeJared) { JarEntry jarAdd = new JarEntry(path); File f = new File(tempDir, path); jarAdd.setTime(now); out.putNextEntry(jarAdd); if (f.isFile()) { FileInputStream in = new FileInputStream(f); try { while (true) { int nRead = in.read(buffer, 0, buffer.length); if (nRead <= 0) { break; } out.write(buffer, 0, nRead); } } finally { in.close(); } } out.closeEntry(); } } try { Files.move(tmpjar, archiveFile); } catch (IOException e) { throw new IOException("Cannot replace " + archiveFile, e); }
4737 1242678047GerritCodeReview/gerritDave Borowitz9e158756e51480dcf145bddedb2286d2c4218850 None In theory we could still unload the plugin even if the rename failed. However, it would be reloaded on the next server startup, which is probably not what the user expects. SATD_ADDED disablePlugins(Set) public void disablePlugins(Set names) if (!isRemoteAdminEnabled()) { log.warn("Remote plugin administration is disabled," + " ignoring disablePlugins(" + names + ")"); return; } synchronized (this) { for (String name : names) { Plugin active = running.get(name); if (active == null) { continue; } log.info(String.format("Disabling plugin %s", active.getName())); Path off = active.getSrcFile().resolveSibling(active.getSrcFile().getFileName() + ".disabled"); try { Files.move(active.getSrcFile(), off); } catch (IOException e) { log.error("Failed to disable plugin", e); // In theory we could still unload the plugin even if the rename // failed. However, it would be reloaded on the next server startup, // which is probably not what the user expects. continue; } unloadPlugin(active); try { FileSnapshot snapshot = FileSnapshot.save(off.toFile()); Plugin offPlugin = loadPlugin(name, off, snapshot); disabled.put(name, offPlugin); } catch (Throwable e) { // This shouldn't happen, as the plugin was loaded earlier. log.warn(String.format("Cannot load disabled plugin %s", active.getName()), e.getCause()); } } cleanInBackground(); }
4734 1242678046GerritCodeReview/gerritDave Borowitz07b40724b5754f2c5eacccc428b7b90660765285 None TODO(dborowitz): Rename to srcPath or something. SATD_ADDED getCleanupHandle() public CleanupHandle getCleanupHandle() return cleanupHandle;
4733 1242678045GerritCodeReview/gerritDave Borowitz32218e3f3407f221c2faa231f521a76fc81f81fd None TODO(dborowitz): Is there a portable way to do this with NIO? SATD_ADDED chmod(int, Path) public static void chmod(final int mode, final Path path) // TODO(dborowitz): Is there a portable way to do this with NIO? chmod(mode, path.toFile());
4732 1242678044GerritCodeReview/gerritDave Borowitz32218e3f3407f221c2faa231f521a76fc81f81fd None TODO(dborowitz): Convert to NIO. SATD_ADDED handles(File) public boolean handles(File srcFile) String fileName = srcFile.getName(); return fileName.endsWith(JAR_EXTENSION) || fileName.endsWith(JAR_EXTENSION + ".disabled");
4731 1242678043GerritCodeReview/gerritDave Borowitz8e481f3c3383dcd4c47335c180d9a64c5b92acad None TODO(dborowitz): Convert all of these to Paths. SATD_ADDED resolve(String) public File resolve(final String path) if (path != null && !path.isEmpty()) { File loc = new File(path); if (!loc.isAbsolute()) { loc = new File(site_path, path); } try { return loc.getCanonicalFile(); } catch (IOException e) { return loc.getAbsoluteFile(); } } return null;
4730 1242678042GerritCodeReview/gerritDave Borowitz8e481f3c3383dcd4c47335c180d9a64c5b92acad None TODO(dborowitz): Use jimfs. SATD_ADDED configure() protected void configure() // For simplicity, don't create child injectors, just use this one to get a // few required modules. Injector cfgInjector = Guice.createInjector(new AbstractModule() { @Override protected void configure() { bind(Config.class).annotatedWith(GerritServerConfig.class).toInstance(cfg); } }); install(cfgInjector.getInstance(GerritGlobalModule.class)); bindScope(RequestScoped.class, PerThreadRequestScope.REQUEST); // TODO(dborowitz): Use jimfs. bind(Path.class).annotatedWith(SitePath.class).toInstance(Paths.get(".")); bind(Config.class).annotatedWith(GerritServerConfig.class).toInstance(cfg); bind(SocketAddress.class).annotatedWith(RemotePeer.class).toInstance(new InetSocketAddress(InetAddresses.forString("127.0.0.1"), 1234)); bind(PersonIdent.class).annotatedWith(GerritPersonIdent.class).toProvider(GerritPersonIdentProvider.class); bind(String.class).annotatedWith(AnonymousCowardName.class).toProvider(AnonymousCowardNameProvider.class); bind(AllProjectsName.class).toProvider(AllProjectsNameProvider.class); bind(AllUsersName.class).toProvider(AllUsersNameProvider.class); bind(GitRepositoryManager.class).to(InMemoryRepositoryManager.class); bind(InMemoryRepositoryManager.class).in(SINGLETON); bind(TrackingFooters.class).toProvider(TrackingFootersProvider.class).in(SINGLETON); bind(DataSourceType.class).to(InMemoryH2Type.class); bind(new TypeLiteral>() { }).to(InMemoryDatabase.class); bind(SecureStore.class).to(DefaultSecureStore.class); bind(ChangeHooks.class).to(DisabledChangeHooks.class); install(NoSshKeyCache.module()); install(new CanonicalWebUrlModule() { @Override protected Class> provider() { return CanonicalWebUrlProvider.class; } }); install(new DefaultCacheFactory.Module()); install(new SmtpEmailSender.Module()); install(new SignedTokenEmailTokenVerifier.Module()); IndexType indexType = null; try { indexType = cfg.getEnum("index", null, "type", IndexType.LUCENE); } catch (IllegalArgumentException e) { // Custom index type, caller must provide their own module. } if (indexType != null) { switch(indexType) { case LUCENE: install(luceneIndexModule()); break; default: throw new ProvisionException("index type unsupported in tests: " + indexType); } }
4726 1242678040GerritCodeReview/gerritDave Borowitz874aed757016989ea2d9ccc360852defa1c6c991 None probably a SHA1 SATD_ADDED apply(RevisionResource, RebaseInput) public ChangeInfo apply(RevisionResource rsrc, RebaseInput input) throws AuthException, ResourceNotFoundException, ResourceConflictException, EmailException, OrmException ChangeControl control = rsrc.getControl(); Change change = rsrc.getChange(); if (!control.canRebase()) { throw new AuthException("rebase not permitted"); } else if (!change.getStatus().isOpen()) { throw new ResourceConflictException("change is " + change.getStatus().name().toLowerCase()); } else if (!hasOneParent(rsrc.getPatchSet().getId())) { throw new ResourceConflictException("cannot rebase merge commits or commit with no ancestor"); } String baseRev = null; if (input != null && input.base != null) { String base = input.base.trim(); do { if (base.equals("")) { // remove existing dependency to other patch set baseRev = change.getDest().get(); break; } ReviewDb db = dbProvider.get(); PatchSet basePatchSet = parseBase(base); if (basePatchSet == null) { throw new ResourceConflictException("base revision is missing: " + base); } else if (!rsrc.getControl().isPatchVisible(basePatchSet, db)) { throw new AuthException("base revision not accessible: " + base); } else if (change.getId().equals(basePatchSet.getId().getParentKey())) { throw new ResourceConflictException("cannot depend on self"); } Change baseChange = db.changes().get(basePatchSet.getId().getParentKey()); if (baseChange != null) { if (!baseChange.getProject().equals(change.getProject())) { throw new ResourceConflictException("base change is in wrong project: " + baseChange.getProject()); } else if (!baseChange.getDest().equals(change.getDest())) { throw new ResourceConflictException("base change is targetting wrong branch: " + baseChange.getDest()); } else if (baseChange.getStatus() == Status.ABANDONED) { throw new ResourceConflictException("base change is abandoned: " + baseChange.getKey()); } baseRev = basePatchSet.getRevision().get(); break; } } while (// just wanted to use the break statement false); } try { rebaseChange.get().rebase(rsrc.getChange(), rsrc.getPatchSet().getId(), rsrc.getUser(), baseRev); } catch (InvalidChangeOperationException e) { throw new ResourceConflictException(e.getMessage()); } catch (IOException e) { throw new ResourceConflictException(e.getMessage()); } catch (NoSuchChangeException e) { throw new ResourceNotFoundException(change.getId().toString()); } return json.format(change.getId());
4724 1242678038GerritCodeReview/gerritStefan Bellerb9db274286c15d1fc26638f602e95baf773b94fb None Properly render revision actions initially while waiting for the callback to populate them correctly. SATD_ADDED renderChangeInfo(ChangeInfo) private void renderChangeInfo(ChangeInfo info) changeInfo = info; lastDisplayedUpdate = info.updated(); labels.set(info); renderOwner(info); renderActionTextDate(info); renderDiffBaseListBox(info); initReplyButton(info, revision); initIncludedInAction(info); initChangeAction(info); initDownloadAction(info, revision); initProjectLinks(info); initBranchLink(info); initEditMode(info, revision); actions.display(info, revision); star.setValue(info.starred()); permalink.setHref(ChangeLink.permalink(changeId)); permalink.setText(String.valueOf(info.legacy_id())); topic.set(info, revision); commit.set(commentLinkProcessor, info, revision); related.set(info, revision); reviewers.set(info); if (Gerrit.isNoteDbEnabled()) { hashtags.set(info); } else { setVisible(hashtagTableRow, false); } StringBuilder sb = new StringBuilder(); sb.append(Util.M.changeScreenTitleId(info.id_abbreviated())); if (info.subject() != null) { sb.append(": "); sb.append(info.subject()); } setWindowTitle(sb.toString()); // Properly render revision actions initially while waiting for // the callback to populate them correctly. renderRevisionInfo(changeInfo, NativeMap.create());
4723 1242678037GerritCodeReview/gerritStefan Beller09cd95dbfd97232df465801e1d6325bbe2ed5898 None TODO(sbeller): why do we need to treat followup specially here? SATD_ADDED ActionInfo> toActionMap(ChangeControl) private Map toActionMap(ChangeControl ctl) Map out = new LinkedHashMap<>(); if (!ctl.getCurrentUser().isIdentifiedUser()) { return out; } Provider userProvider = Providers.of(ctl.getCurrentUser()); for (UiAction.Description d : UiActions.from(changeViews, new ChangeResource(ctl), userProvider)) { out.put(d.getId(), new ActionInfo(d)); } // TODO(sbeller): why do we need to treat followup specially here? if (ctl.getChange().getStatus().isOpen()) { UiAction.Description descr = new UiAction.Description(); PrivateInternals_UiActionDescription.setId(descr, "followup"); PrivateInternals_UiActionDescription.setMethod(descr, "POST"); descr.setTitle("Create follow-up change"); out.put(descr.getId(), new ActionInfo(descr)); } return out;
4721 1242678036GerritCodeReview/gerritDavid Pursehouse875fc8e8a3367d2507d0f937e13b12d3ff3f6f86 None TODO(dborowitz): Implement RefUpdate.toString(). SATD_ADDED updateBranch(SubmitStrategy, RefUpdate) private RefUpdate updateBranch(SubmitStrategy strategy, RefUpdate branchUpdate) throws MergeException CodeReviewCommit currentTip = mergeTip != null ? mergeTip.getCurrentTip() : null; if (Objects.equals(branchTip, currentTip)) { logDebug("Branch already at merge tip {}, no update to perform", currentTip.name()); return null; } else if (currentTip == null) { logDebug("No merge tip, no update to perform"); return null; } if (RefNames.REFS_CONFIG.equals(branchUpdate.getName())) { logDebug("Loading new configuration from {}", RefNames.REFS_CONFIG); try { ProjectConfig cfg = new ProjectConfig(destProject.getProject().getNameKey()); cfg.load(repo, currentTip); } catch (Exception e) { throw new MergeException("Submit would store invalid" + " project configuration " + currentTip.name() + " for " + destProject.getProject().getName(), e); } } branchUpdate.setRefLogIdent(refLogIdent); branchUpdate.setForceUpdate(false); branchUpdate.setNewObjectId(currentTip); branchUpdate.setRefLogMessage("merged", true); try { RefUpdate.Result result = branchUpdate.update(rw); logDebug("Update of {}: {}..{} returned status {}", branchUpdate.getName(), branchUpdate.getOldObjectId(), branchUpdate.getNewObjectId(), result); switch(result) { case NEW: case FAST_FORWARD: if (branchUpdate.getResult() == RefUpdate.Result.FAST_FORWARD) { tagCache.updateFastForward(destBranch.getParentKey(), branchUpdate.getName(), branchUpdate.getOldObjectId(), currentTip); } if (RefNames.REFS_CONFIG.equals(branchUpdate.getName())) { Project p = destProject.getProject(); projectCache.evict(p); destProject = projectCache.get(p.getNameKey()); repoManager.setProjectDescription(p.getNameKey(), p.getDescription()); } return branchUpdate; case LOCK_FAILURE: String msg; if (strategy.retryOnLockFailure()) { mergeQueue.recheckAfter(destBranch, LOCK_FAILURE_RETRY_DELAY, MILLISECONDS); msg = "will retry"; } else { msg = "will not retry"; } // TODO(dborowitz): Implement RefUpdate.toString(). throw new IOException(branchUpdate.getResult().name() + ", " + msg + '\n' + branchUpdate); default: throw new IOException(branchUpdate.getResult().name() + '\n' + branchUpdate); } } catch (IOException e) { throw new MergeException("Cannot update " + branchUpdate.getName(), e); }
4719 1242678035GerritCodeReview/gerritSaša Živkovc20148c8e20d62744c3654e73fb9d7256f30a205 None TODO(dborowitz): This message is also used by all new patch set ref SATD_ADDED updateBranch(SubmitStrategy, BatchRefUpdate) private BatchRefUpdate updateBranch(SubmitStrategy strategy, BatchRefUpdate branchUpdate) throws MergeException CodeReviewCommit currentTip = mergeTip != null ? mergeTip.getCurrentTip() : null; if (branchTip == currentTip) { logDebug("Branch already at merge tip {}, no update to perform", currentTip.name()); return null; } else if (currentTip == null) { logDebug("No merge tip, no update to perform"); return null; } if (RefNames.REFS_CONFIG.equals(destBranch.get())) { logDebug("Loading new configuration from {}", RefNames.REFS_CONFIG); try { ProjectConfig cfg = new ProjectConfig(destProject.getProject().getNameKey()); cfg.load(repo, currentTip); } catch (Exception e) { throw new MergeException("Submit would store invalid" + " project configuration " + currentTip.name() + " for " + destProject.getProject().getName(), e); } } try { inserter.flush(); } catch (IOException e) { throw new MergeException("Cannot flush merge results", e); } branchUpdate.setRefLogIdent(refLogIdent); branchUpdate.setAllowNonFastForwards(false); // TODO(dborowitz): This message is also used by all new patch set ref // updates; find a better wording that works for that case too. branchUpdate.setRefLogMessage("merged", true); ReceiveCommand cmd = new ReceiveCommand(oldBranchTip, currentTip, destBranch.get()); branchUpdate.addCommand(cmd); try { branchUpdate.execute(rw, NullProgressMonitor.INSTANCE); logDebug("Executed batch update: {}", branchUpdate); switch(cmd.getResult()) { case OK: if (cmd.getType() == ReceiveCommand.Type.UPDATE) { tagCache.updateFastForward(destBranch.getParentKey(), destBranch.get(), oldBranchTip, currentTip); } if (RefNames.REFS_CONFIG.equals(destBranch.get())) { Project p = destProject.getProject(); projectCache.evict(p); destProject = projectCache.get(p.getNameKey()); repoManager.setProjectDescription(p.getNameKey(), p.getDescription()); } return branchUpdate; case LOCK_FAILURE: String msg; if (strategy.retryOnLockFailure()) { mergeQueue.recheckAfter(destBranch, LOCK_FAILURE_RETRY_DELAY, MILLISECONDS); msg = "will retry"; } else { msg = "will not retry"; } throw new IOException(cmd.getResult().name() + ", " + msg); default: throw new IOException(cmd.getResult().name()); } } catch (IOException e) { throw new MergeException("Cannot update " + destBranch.get(), e); }
4716 1242678033GerritCodeReview/gerritDavid Myllykangas45b0191c44a7d95d7219649c39eb4abc028c0afe None ignore non valid numbers We don't want to popup another ugly dialog just to say \"The number you've provided is invalid, try again\" SATD_ADDED gotoLine() private Runnable gotoLine() return new Runnable() { @Override public void run() { String n = Window.prompt(EditConstants.I.gotoLineNumber(), ""); if (n != null) { try { int line = Integer.parseInt(n); line--; if (line >= 0) { cm.scrollToLine(line); } } catch (NumberFormatException e) { // ignore non valid numbers // We don't want to popup another ugly dialog just to say // "The number you've provided is invalid, try again" } } } };
4715 1242678032GerritCodeReview/gerritDave Borowitzdfc07f63bb21b4c1c9296cb88ac56f536bb92ddf None TODO(dborowitz): Consider putting ChangeData in CodeReviewCommit. SATD_ADDED Change> validateChangeList(List) private ListMultimap validateChangeList(List submitted) throws MergeException ListMultimap toSubmit = ArrayListMultimap.create(); Map allRefs; try { allRefs = repo.getRefDatabase().getRefs(ALL); } catch (IOException e) { throw new MergeException(e.getMessage(), e); } Set tips = new HashSet<>(); for (Ref r : allRefs.values()) { tips.add(r.getObjectId()); } for (ChangeData cd : submitted) { ChangeControl ctl; Change chg; try { ctl = cd.changeControl(); chg = cd.change(); } catch (OrmException e) { throw new MergeException("Failed to validate changes", e); } Change.Id changeId = cd.getId(); if (chg.currentPatchSetId() == null) { logError("Missing current patch set on change " + changeId); commits.put(changeId, CodeReviewCommit.noPatchSet(ctl)); toUpdate.add(chg); continue; } PatchSet ps; try { ps = cd.currentPatchSet(); } catch (OrmException e) { throw new MergeException("Cannot query the database", e); } if (ps == null || ps.getRevision() == null || ps.getRevision().get() == null) { logError("Missing patch set or revision on change " + changeId); commits.put(changeId, CodeReviewCommit.noPatchSet(ctl)); toUpdate.add(chg); continue; } String idstr = ps.getRevision().get(); ObjectId id; try { id = ObjectId.fromString(idstr); } catch (IllegalArgumentException iae) { logError("Invalid revision on patch set " + ps.getId()); commits.put(changeId, CodeReviewCommit.noPatchSet(ctl)); toUpdate.add(chg); continue; } if (!tips.contains(id)) { // TODO Technically the proper way to do this test is to use a // RevWalk on "$id --not --all" and test for an empty set. But // that is way slower than looking for a ref directly pointing // at the desired tip. We should always have a ref available. // // TODO this is actually an error, the branch is gone but we // want to merge the issue. We can't safely do that if the // tip is not reachable. // logError("Revision " + idstr + " of patch set " + ps.getId() + " is not contained in any ref"); commits.put(changeId, CodeReviewCommit.revisionGone(ctl)); toUpdate.add(chg); continue; } CodeReviewCommit commit; try { commit = (CodeReviewCommit) rw.parseCommit(id); } catch (IOException e) { logError("Invalid commit " + idstr + " on patch set " + ps.getId(), e); commits.put(changeId, CodeReviewCommit.revisionGone(ctl)); toUpdate.add(chg); continue; } // TODO(dborowitz): Consider putting ChangeData in CodeReviewCommit. commit.setControl(ctl); commit.setPatchsetId(ps.getId()); commits.put(changeId, commit); MergeValidators mergeValidators = mergeValidatorsFactory.create(); try { mergeValidators.validatePreMerge(repo, commit, destProject, destBranch, ps.getId()); } catch (MergeValidationException mve) { logDebug("Revision {} of patch set {} failed validation: {}", idstr, ps.getId(), mve.getStatus()); commit.setStatusCode(mve.getStatus()); toUpdate.add(chg); continue; } if (branchTip != null) { // If this commit is already merged its a bug in the queuing code // that we got back here. Just mark it complete and move on. It's // merged and that is all that mattered to the requestor. // try { if (rw.isMergedInto(commit, branchTip)) { logDebug("Revision {} of patch set {} is already merged", idstr, ps.getId()); commit.setStatusCode(CommitMergeStatus.ALREADY_MERGED); try { setMerged(chg, null); } catch (OrmException e) { logError("Cannot mark change " + chg.getId() + " merged", e); } continue; } } catch (IOException err) { throw new MergeException("Cannot perform merge base test", err); } } SubmitType submitType; submitType = getSubmitType(commit.getControl(), ps); if (submitType == null) { logError("No submit type for revision " + idstr + " of patch set " + ps.getId()); commit.setStatusCode(CommitMergeStatus.NO_SUBMIT_TYPE); toUpdate.add(chg); continue; } commit.add(canMergeFlag); toMerge.put(submitType, commit); toSubmit.put(submitType, chg); } return toSubmit;
4714 1242678031GerritCodeReview/gerritDave Borowitzdfc07f63bb21b4c1c9296cb88ac56f536bb92ddf None TODO(dborowitz): Not sure whether this is injectable here. SATD_ADDED parseArguments(Parameters) public final int parseArguments(final Parameters params) throws CmdLineException final String token = params.getParameter(0); final String[] tokens = token.split(","); if (tokens.length != 3) { throw new CmdLineException(owner, "change should be specified as " + ",,"); } try { final Change.Key key = Change.Key.parse(tokens[2]); final Project.NameKey project = new Project.NameKey(tokens[0]); final Branch.NameKey branch = new Branch.NameKey(project, "refs/heads/" + tokens[1]); for (final ChangeData cd : queryProvider.get().byBranchKey(branch, key)) { setter.addValue(cd.getId()); return 1; } } catch (IllegalArgumentException e) { throw new CmdLineException(owner, "Change-Id is not valid"); } catch (OrmException e) { throw new CmdLineException(owner, "Database error: " + e.getMessage()); } throw new CmdLineException(owner, "\"" + token + "\": change not found");
4713 1242677748GerritCodeReview/gerritDave Borowitzdfc07f63bb21b4c1c9296cb88ac56f536bb92ddf TODO Technically the proper way to do this test is to use a RevWalk on \"$id --not --all\" and test for an empty set. But that is way slower than looking for a ref directly pointing at the desired tip. We should always have a ref available. TODO this is actually an error, the branch is gone but we want to merge the issue. We can't safely do that if the tip is not reachable. TODO Technically the proper way to do this test is to use a RevWalk on \"$id --not --all\" and test for an empty set. But that is way slower than looking for a ref directly pointing at the desired tip. We should always have a ref available. TODO this is actually an error, the branch is gone but we want to merge the issue. We can't safely do that if the tip is not reachable. CLASS_OR_METHOD_CHANGED Change> validateChangeList(List) private ListMultimap validateChangeList(List submitted) throws MergeException ListMultimap toSubmit = ArrayListMultimap.create(); Map allRefs; try { allRefs = repo.getRefDatabase().getRefs(ALL); } catch (IOException e) { throw new MergeException(e.getMessage(), e); } Set tips = new HashSet<>(); for (Ref r : allRefs.values()) { tips.add(r.getObjectId()); } for (ChangeData cd : submitted) { ChangeControl ctl; Change chg; try { ctl = cd.changeControl(); chg = cd.change(); } catch (OrmException e) { throw new MergeException("Failed to validate changes", e); } Change.Id changeId = cd.getId(); if (chg.currentPatchSetId() == null) { logError("Missing current patch set on change " + changeId); commits.put(changeId, CodeReviewCommit.noPatchSet(ctl)); toUpdate.add(chg); continue; } PatchSet ps; try { ps = cd.currentPatchSet(); } catch (OrmException e) { throw new MergeException("Cannot query the database", e); } if (ps == null || ps.getRevision() == null || ps.getRevision().get() == null) { logError("Missing patch set or revision on change " + changeId); commits.put(changeId, CodeReviewCommit.noPatchSet(ctl)); toUpdate.add(chg); continue; } String idstr = ps.getRevision().get(); ObjectId id; try { id = ObjectId.fromString(idstr); } catch (IllegalArgumentException iae) { logError("Invalid revision on patch set " + ps.getId()); commits.put(changeId, CodeReviewCommit.noPatchSet(ctl)); toUpdate.add(chg); continue; } if (!tips.contains(id)) { // TODO Technically the proper way to do this test is to use a // RevWalk on "$id --not --all" and test for an empty set. But // that is way slower than looking for a ref directly pointing // at the desired tip. We should always have a ref available. // // TODO this is actually an error, the branch is gone but we // want to merge the issue. We can't safely do that if the // tip is not reachable. // logError("Revision " + idstr + " of patch set " + ps.getId() + " is not contained in any ref"); commits.put(changeId, CodeReviewCommit.revisionGone(ctl)); toUpdate.add(chg); continue; } CodeReviewCommit commit; try { commit = (CodeReviewCommit) rw.parseCommit(id); } catch (IOException e) { logError("Invalid commit " + idstr + " on patch set " + ps.getId(), e); commits.put(changeId, CodeReviewCommit.revisionGone(ctl)); toUpdate.add(chg); continue; } // TODO(dborowitz): Consider putting ChangeData in CodeReviewCommit. commit.setControl(ctl); commit.setPatchsetId(ps.getId()); commits.put(changeId, commit); MergeValidators mergeValidators = mergeValidatorsFactory.create(); try { mergeValidators.validatePreMerge(repo, commit, destProject, destBranch, ps.getId()); } catch (MergeValidationException mve) { logDebug("Revision {} of patch set {} failed validation: {}", idstr, ps.getId(), mve.getStatus()); commit.setStatusCode(mve.getStatus()); toUpdate.add(chg); continue; } if (branchTip != null) { // If this commit is already merged its a bug in the queuing code // that we got back here. Just mark it complete and move on. It's // merged and that is all that mattered to the requestor. // try { if (rw.isMergedInto(commit, branchTip)) { logDebug("Revision {} of patch set {} is already merged", idstr, ps.getId()); commit.setStatusCode(CommitMergeStatus.ALREADY_MERGED); try { setMerged(chg, null); } catch (OrmException e) { logError("Cannot mark change " + chg.getId() + " merged", e); } continue; } } catch (IOException err) { throw new MergeException("Cannot perform merge base test", err); } } SubmitType submitType; submitType = getSubmitType(commit.getControl(), ps); if (submitType == null) { logError("No submit type for revision " + idstr + " of patch set " + ps.getId()); commit.setStatusCode(CommitMergeStatus.NO_SUBMIT_TYPE); toUpdate.add(chg); continue; } commit.add(canMergeFlag); toMerge.put(submitType, commit); toSubmit.put(submitType, chg); } return toSubmit;
4712 1242677951GerritCodeReview/gerritShawn Pearce0ddfb83db9c7993d8d1b16837d20615a11c0fd6c TODO info card hover TODO info card hover FILE_PATH_CHANGED renderOwner(ChangeInfo) private void renderOwner(ChangeInfo info) // TODO info card hover String name = info.owner().name() != null ? info.owner().name() : Gerrit.getConfig().getAnonymousCowardName(); if (info.owner().avatar(AvatarInfo.DEFAULT_SIZE) != null) { ownerPanel.insert(new AvatarImage(info.owner()), 0); } ownerLink.setText(name); ownerLink.setTitle(info.owner().email() != null ? info.owner().email() : name); ownerLink.setTargetHistoryToken(PageLinks.toAccountQuery(info.owner().name() != null ? info.owner().name() : info.owner().email() != null ? info.owner().email() : String.valueOf(info.owner()._account_id()), Change.Status.NEW));
4711 1242677766GerritCodeReview/gerritShawn Pearcea386fb30356bb3ecd0cbc11b9a23d13e7aa6ce5f Green check mark if the user is logged in and they reviewed that file Green check mark if the user is logged in and they reviewed that file FILE_PATH_CHANGED isValid(Patch) mp boolean isValid(Patch patch)
4708 1242677917GerritCodeReview/gerritDave Borowitz53a5ccd4df2f86c75a2023fff3206c276adb996f TODO: propagate this exception TODO: propagate this exception CLASS_OR_METHOD_CHANGED controlFor(Change.Id, CurrentUser) public ChangeControl controlFor(Change.Id changeId, CurrentUser user) throws NoSuchChangeException, OrmException Change change = db.get().changes().get(changeId); if (change == null) { throw new NoSuchChangeException(changeId); } return controlFor(change, user);
4707 1242677966GerritCodeReview/gerritDave Borowitzcf4ff534fffb3bae2833a5d174077a029119ac28 we don't care better we would check if index was already renamed gwtorm doesn't expose this functionality gwtorm didn't expose this functionality at the time this schema upgrade was written. SATD_REMOVED preUpdateSchema(ReviewDb) protected void preUpdateSchema(ReviewDb db) throws OrmException, SQLException JdbcSchema s = (JdbcSchema) db; try (JdbcExecutor e = new JdbcExecutor(s)) { renameTables(db, s, e); renameColumn(db, s, e); } renameIndexes(db);
4703 1242678030GerritCodeReview/gerritShawn Pearce6cadc12269fbee38252a4ec4ecedd08b1254b7b7 TODO: Better custom keybindings, remove temporary navigation hacks. TODO: Better custom keybindings, remove temporary navigation hacks. SATD_MOVED_FILE initKeyMap() static void initKeyMap() // TODO: Better custom keybindings, remove temporary navigation hacks. KeyMap km = CodeMirror.cloneKeyMap("vim"); for (String s : new String[] { "A", "C", "I", "O", "R", "U", "Ctrl-C", "Ctrl-O", "Ctrl-P", "Ctrl-S", "Ctrl-F", "Ctrl-B", "Ctrl-R" }) { km.remove(s); } for (int i = 0; i <= 9; i++) { km.remove("Ctrl-" + i); } CodeMirror.addKeyMap("vim_ro", km); mapKey("j", "gj"); mapKey("k", "gk"); mapKey("Down", "gj"); mapKey("Up", "gk"); mapKey("", ""); mapKey("", "");
4702 1242678029GerritCodeReview/gerritDave Borowitz6c69f8435a835e30ed8425c17ae1fb37a7b3838a Always bump limit by 1, even if this results in exceeding the permitted max for this user. The only way to see if there are more changes is to ask for one more result from the query. Always bump limit by 1, even if this results in exceeding the permitted max for this user. The only way to see if there are more changes is to ask for one more result from the query. CLASS_OR_METHOD_CHANGED queryChanges(List, List>) private List queryChanges(List queryStrings, List> queries) throws OrmException, QueryParseException Predicate visibleToMe = queryBuilder.is_visible(); int cnt = queries.size(); // Parse and rewrite all queries. List limits = new ArrayList<>(cnt); List> predicates = new ArrayList<>(cnt); List sources = new ArrayList<>(cnt); for (Predicate q : queries) { q = Predicate.and(q, visibleToMe); int limit = getEffectiveLimit(q); limits.add(limit); // Always bump limit by 1, even if this results in exceeding the permitted // max for this user. The only way to see if there are more changes is to // ask for one more result from the query. Predicate s = queryRewriter.rewrite(q, start, limit + 1); if (!(s instanceof ChangeDataSource)) { q = Predicate.and(queryBuilder.status_open(), q); s = queryRewriter.rewrite(q, start, limit); } if (!(s instanceof ChangeDataSource)) { throw new QueryParseException("invalid query: " + s); } predicates.add(s); // Don't trust QueryRewriter to have left the visible predicate. // TODO(dborowitz): Probably we can. AndSource a = new AndSource(ImmutableList.of(s, visibleToMe), start); sources.add(a); } // Run each query asynchronously, if supported. List> matches = new ArrayList<>(cnt); for (ChangeDataSource s : sources) { matches.add(s.read()); } List out = new ArrayList<>(cnt); for (int i = 0; i < cnt; i++) { out.add(QueryResult.create(queryStrings != null ? queryStrings.get(i) : null, predicates.get(i), limits.get(i), matches.get(i).toList())); } return out;
4701 1242678029GerritCodeReview/gerritDave Borowitzdb52c00bc382d9addebbc3e7b4dcf2152db16a36 None Always bump limit by 1, even if this results in exceeding the permitted max for this user. The only way to see if there are more changes is to ask for one more result from the query. SATD_ADDED setLimit(int) void setLimit(int n) limitFromCaller = n;
4700 1242678028GerritCodeReview/gerritDavid Ostrovsky8ffdfc124364ab284cef8999631f0e2a0812d064 None TODO(davido): Retrieve user preferences from AllUsers repository SATD_ADDED getConfig() private Configuration getConfig() // TODO(davido): Retrieve user preferences from AllUsers repository return Configuration.create().set("readOnly", false).set("cursorBlinkRate", 0).set("cursorHeight", 0.85).set("lineNumbers", true).set("tabSize", 4).set("lineWrapping", false).set("styleSelectedText", true).set("showTrailingSpace", true).set("keyMap", "default").set("mode", ModeInjector.getContentType(type));
4696 1242678025GerritCodeReview/gerritDavid Ostrovsky11c56681b01a39c109a406f78cfcc2c59a8ffdc8 None TODO: why do we need this? Looks like Ray added it a long time ago. SATD_ADDED start(TreeLogger) void start(final TreeLogger logger) throws UnableToCompleteException Server newServer = new Server(); ServerConnector connector = new ServerConnector(newServer); connector.setHost(bindAddress); connector.setPort(port); connector.setReuseAddress(false); connector.setSoLingerTime(0); newServer.addConnector(connector); ServletContextHandler newHandler = new ServletContextHandler(ServletContextHandler.SESSIONS); newHandler.setContextPath("/"); newHandler.addServlet(new ServletHolder(new HttpServlet() { @Override protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { handleRequest(request.getPathInfo(), request, response, logger); } }), "/*"); newHandler.addFilter(GzipFilter.class, "/*", EnumSet.allOf(DispatcherType.class)); newServer.setHandler(newHandler); try { newServer.start(); } catch (Exception e) { logger.log(TreeLogger.ERROR, "cannot start web server", e); throw new UnableToCompleteException(); } this.server = newServer;
4695 1242678024GerritCodeReview/gerritDavid Ostrovsky11c56681b01a39c109a406f78cfcc2c59a8ffdc8 None TODO: return a list of progress objects here, one for each job. SATD_ADDED start(TreeLogger) void start(final TreeLogger logger) throws UnableToCompleteException Server newServer = new Server(); ServerConnector connector = new ServerConnector(newServer); connector.setHost(bindAddress); connector.setPort(port); connector.setReuseAddress(false); connector.setSoLingerTime(0); newServer.addConnector(connector); ServletContextHandler newHandler = new ServletContextHandler(ServletContextHandler.SESSIONS); newHandler.setContextPath("/"); newHandler.addServlet(new ServletHolder(new HttpServlet() { @Override protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { handleRequest(request.getPathInfo(), request, response, logger); } }), "/*"); newHandler.addFilter(GzipFilter.class, "/*", EnumSet.allOf(DispatcherType.class)); newServer.setHandler(newHandler); try { newServer.start(); } catch (Exception e) { logger.log(TreeLogger.ERROR, "cannot start web server", e); throw new UnableToCompleteException(); } this.server = newServer;
4694 1242678010GerritCodeReview/gerritDavid Ostrovsky11c56681b01a39c109a406f78cfcc2c59a8ffdc8 perhaps it's compressed perhaps it's compressed CLASS_OR_METHOD_CHANGED start(TreeLogger) void start(final TreeLogger logger) throws UnableToCompleteException Server newServer = new Server(); ServerConnector connector = new ServerConnector(newServer); connector.setHost(bindAddress); connector.setPort(port); connector.setReuseAddress(false); connector.setSoLingerTime(0); newServer.addConnector(connector); ServletContextHandler newHandler = new ServletContextHandler(ServletContextHandler.SESSIONS); newHandler.setContextPath("/"); newHandler.addServlet(new ServletHolder(new HttpServlet() { @Override protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { handleRequest(request.getPathInfo(), request, response, logger); } }), "/*"); newHandler.addFilter(GzipFilter.class, "/*", EnumSet.allOf(DispatcherType.class)); newServer.setHandler(newHandler); try { newServer.start(); } catch (Exception e) { logger.log(TreeLogger.ERROR, "cannot start web server", e); throw new UnableToCompleteException(); } this.server = newServer;
4693 1242678014GerritCodeReview/gerritEdwin Kempinb7da2a97d4798de64a25354e9484e152a383094e None Previously there was a bug where having a '1' in a refname would cause a glob pattern's Levenshtein distance to decrease by 1. These two patterns should be a Levenshtein distance of 12 from the both of the refnames, where previously the 'branch1' refname would be a distance of 11 from 'refs/heads/abc/*' SATD_ADDED higherNumberOfTransitionsWins() public void higherNumberOfTransitionsWins() cmp = new MostSpecificComparator("refs/heads/x"); moreSpecificFirst("^refs/heads/[a-z].*", "refs/heads/*"); // Previously there was a bug where having a '1' in a refname would cause a // glob pattern's Levenshtein distance to decrease by 1. These two // patterns should be a Levenshtein distance of 12 from the both of the // refnames, where previously the 'branch1' refname would be a distance of // 11 from 'refs/heads/abc/*' cmp = new MostSpecificComparator("refs/heads/abc/spam/branch2"); moreSpecificFirst("^refs/heads/.*spam.*", "refs/heads/abc/*"); cmp = new MostSpecificComparator("refs/heads/abc/spam/branch1"); moreSpecificFirst("^refs/heads/.*spam.*", "refs/heads/abc/*");
4692 1242678023GerritCodeReview/gerritDavid Pursehouse4248ea8c8626c75995f116bac043d4c04fd24879 Does not account for draft status as a user might want to let a reviewer see a draft. Does not account for draft status as a user might want to let a reviewer see a draft. CLASS_OR_METHOD_CHANGED isValidReviewer(Account, ChangeControl) private boolean isValidReviewer(Account member, ChangeControl control) if (member.isActive()) { IdentifiedUser user = identifiedUserFactory.create(member.getId()); // Does not account for draft status as a user might want to let a // reviewer see a draft. return control.forUser(user).isRefVisible(); } return false;
4689 1242678022GerritCodeReview/gerritDavid Ostrovsky3d2c070206f8b277015a26ac20323f8671933c08 None TODO(davido): Clean this up by returning 404 when edit doesn't exist. SATD_ADDED apply(ChangeResource) public BinaryResult apply(ChangeResource rsrc) throws AuthException, IOException, OrmException, NoSuchChangeException Optional edit = editUtil.byChange(rsrc.getChange()); // TODO(davido): Clean this up by returning 404 when edit doesn't exist. // Client should call GET /changes/{id}/revisions/current/commit in this // case; or, to be consistent with GET content logic, the client could // call directly the right endpoint. String m = edit.isPresent() ? edit.get().getEditCommit().getFullMessage() : changeUtil.getMessage(rsrc.getChange()); return BinaryResult.create(m).base64();
4688 1242678021GerritCodeReview/gerritDave Borowitzfd508cab05da84e9d23d9666d41923511e01f2cf None TODO(dborowitz): Expose stack trace to administrators. SATD_ADDED error(String, Throwable) private boolean error(String msg, Throwable t) messages.add(msg); // TODO(dborowitz): Expose stack trace to administrators. log.warn("Error in consistency check of change " + change.getId(), t); return false;
4687 1242678020GerritCodeReview/gerritDave Borowitzfd508cab05da84e9d23d9666d41923511e01f2cf None TODO(dborowitz): Just fix it. SATD_ADDED checkMerged() private void checkMerged() String refName = change.getDest().get(); Ref dest; try { dest = repo.getRef(refName); } catch (IOException e) { messages.add("Failed to look up destination ref: " + refName); return; } if (dest == null) { messages.add("Destination ref not found (may be new branch): " + change.getDest().get()); return; } RevCommit tip = parseCommit(dest.getObjectId(), "destination ref " + refName); if (tip == null) { return; } boolean merged; try { merged = rw.isMergedInto(currPsCommit, tip); } catch (IOException e) { messages.add("Error checking whether patch set " + currPs.getId().get() + " is merged"); return; } if (merged && change.getStatus() != Change.Status.MERGED) { messages.add(String.format("Patch set %d (%s) is merged into destination" + " ref %s (%s), but change status is %s", currPs.getId().get(), currPsCommit.name(), refName, tip.name(), change.getStatus())); // TODO(dborowitz): Just fix it. } else if (!merged && change.getStatus() == Change.Status.MERGED) { messages.add(String.format("Patch set %d (%s) is not merged into" + " destination ref %s (%s), but change status is %s", currPs.getId().get(), currPsCommit.name(), refName, tip.name(), change.getStatus())); }
4686 1242678019GerritCodeReview/gerritDavid Pursehouse12b44cb3904ee982c47a8cfbf3499f0f174c67b6 None maybe the primary key was dropped in a previous run but the creation failed SATD_ADDED recreatePK(StatementExecutor, String, List) private void recreatePK(StatementExecutor executor, String tableName, List cols) throws OrmException try { if (dialect instanceof DialectPostgreSQL) { // postgresql doesn't support the ALTER TABLE foo DROP PRIMARY KEY form executor.execute("ALTER TABLE " + tableName + " DROP CONSTRAINT " + tableName + "_pkey"); } else { executor.execute("ALTER TABLE " + tableName + " DROP PRIMARY KEY"); } } catch (OrmException ignore) { // maybe the primary key was dropped in a previous run but the creation failed ui.message("WARN: %s\n", ignore.getMessage()); } executor.execute("ALTER TABLE " + tableName + " ADD PRIMARY KEY(" + Joiner.on(",").join(cols) + ")");
4685 1242678018GerritCodeReview/gerritDave Borowitzb9596123373163913b65671029868603a795e0d5 None TODO(dborowitz): Test for other-branches. SATD_ADDED mergeable() public void mergeable() throws Exception ObjectId initial = git.getRepository().getRef(HEAD).getLeaf().getObjectId(); PushOneCommit push1 = pushFactory.create(db, admin.getIdent(), PushOneCommit.SUBJECT, PushOneCommit.FILE_NAME, "push 1 content"); PushOneCommit.Result r1 = push1.to(git, "refs/for/master"); assertMergeable(r1.getChangeId(), true); merge(r1); // Reset HEAD to initial so the new change is a merge conflict. RefUpdate ru = git.getRepository().updateRef(HEAD); ru.setNewObjectId(initial); assertThat(ru.forceUpdate()).isEqualTo(RefUpdate.Result.FORCED); PushOneCommit push2 = pushFactory.create(db, admin.getIdent(), PushOneCommit.SUBJECT, PushOneCommit.FILE_NAME, "push 2 content"); PushOneCommit.Result r2 = push2.to(git, "refs/for/master"); assertMergeable(r2.getChangeId(), false); // TODO(dborowitz): Test for other-branches.
4684 1242678017GerritCodeReview/gerritDave Borowitz1c8767ed1407a910a1b48ac7b46ed617da61d06a None TODO(dborowitz): Include cache info in ETag somehow instead. SATD_ADDED refresh(Change, ObjectId, Ref, SubmitType, String, Repository, Boolean) private boolean refresh(final Change change, ObjectId commit, final Ref ref, SubmitType type, String strategy, Repository git, Boolean old) throws OrmException, IOException final boolean mergeable = cache.get(commit, ref, type, strategy, change.getDest(), git, db.get()); if (!Objects.equals(mergeable, old)) { // TODO(dborowitz): Include cache info in ETag somehow instead. ChangeUtil.bumpRowVersionNotLastUpdatedOn(change.getId(), db.get()); indexer.index(db.get(), change); } return mergeable;
4683 1242677960GerritCodeReview/gerritDave Borowitzdb27fb30648d2790ca304ea51fafe187d40106a4 TODO(dborowitz): Opening all repositories in a live server may be TODO(dborowitz): Opening all repositories in a live server may be FILE_PATH_CHANGED success() public boolean success() return success;
4682 1242677952GerritCodeReview/gerritHugo Arès8b42ec5bd5cd9aa7a6d7b8159ff4a9911ba89175 TODO Replace ActionDialog with a nicer looking display. TODO Replace ActionDialog with a nicer looking display. CLASS_OR_METHOD_CHANGED call(Button, Change.Id, String, String, String) static void call(Button b, final Change.Id id, final String revision, String project, final String commitSubject) // TODO Replace ActionDialog with a nicer looking display. b.setEnabled(false); new ActionDialog(b, false, Util.C.revertChangeTitle(), Util.C.headingRevertMessage()) { { sendButton.setText(Util.C.buttonRevertChangeSend()); message.setText(Util.M.revertChangeDefaultMessage(commitSubject, revision)); } @Override public void onSend() { ChangeApi.revert(id.get(), getMessageText(), new GerritCallback() { @Override public void onSuccess(ChangeInfo result) { sent = true; hide(); Gerrit.display(PageLinks.toChange2(result.legacy_id())); } @Override public void onFailure(Throwable caught) { enableButtons(true); super.onFailure(caught); } }); } }.center();
4680 1242678016GerritCodeReview/gerritDave Borowitz7ebb55d916062bea8181fd96f74b98c47fe3bd85 Should never occur for a well written rule Should never occur for a well written rule SATD_MOVED_FILE getSubmitType() public SubmitTypeRecord getSubmitType() List results; try { results = evaluateImpl("locate_submit_type", "get_submit_type", "locate_submit_type_filter", "filter_submit_type_results"); } catch (RuleEvalException e) { return typeError(e.getMessage(), e); } if (results.isEmpty()) { // Should never occur for a well written rule return typeError("Submit rule '" + getSubmitRule() + "' for change " + cd.getId() + " of " + getProjectName() + " has no solution."); } Term typeTerm = results.get(0); if (!typeTerm.isSymbol()) { return typeError("Submit rule '" + getSubmitRule() + "' for change " + cd.getId() + " of " + getProjectName() + " did not return a symbol."); } String typeName = ((SymbolTerm) typeTerm).name(); try { return SubmitTypeRecord.OK(SubmitType.valueOf(typeName.toUpperCase())); } catch (IllegalArgumentException e) { return typeError("Submit type rule " + getSubmitRule() + " for change " + cd.getId() + " of " + getProjectName() + " output invalid result: " + typeName); }
4679 1242678015GerritCodeReview/gerritDavid Pursehouse775f4db81cb3b10552886700d1a6f9d8e425f40b None TODO(dpursehouse): validate hashtags SATD_ADDED get() public String get() return value;
4678 1242678014GerritCodeReview/gerritDavid Ostrovsky2a53365ce37189542daa18820c68ad94405f619b None Previously there was a bug where having a '1' in a refname would cause a glob pattern's Levenshtein distance to decrease by 1. These two patterns should be a Levenshtein distance of 12 from the both of the refnames, where previously the 'branch1' refname would be a distance of 11 from 'refs/heads/abc/*' SATD_ADDED higherNumberOfTransitionsWins() public void higherNumberOfTransitionsWins() cmp = new MostSpecificComparator("refs/heads/x"); moreSpecificFirst("^refs/heads/[a-z].*", "refs/heads/*"); // Previously there was a bug where having a '1' in a refname would cause a // glob pattern's Levenshtein distance to decrease by 1. These two // patterns should be a Levenshtein distance of 12 from the both of the // refnames, where previously the 'branch1' refname would be a distance of // 11 from 'refs/heads/abc/*' cmp = new MostSpecificComparator("refs/heads/abc/spam/branch2"); moreSpecificFirst("^refs/heads/.*spam.*", "refs/heads/abc/*"); cmp = new MostSpecificComparator("refs/heads/abc/spam/branch1"); moreSpecificFirst("^refs/heads/.*spam.*", "refs/heads/abc/*");
4677 1242678013GerritCodeReview/gerritDave Borowitzd2468d3336846cc47b6ed9275df68f7d7f16b175 None TODO: The user should have create permission on the branch referred to by HEAD. This would have to happen on the server side. SATD_ADDED onInitUI() protected void onInitUI() super.onInitUI(); Resources.I.style().ensureInjected(); saveProject = new Button(Util.C.buttonSaveChanges()); saveProject.addClickHandler(new ClickHandler() { @Override public void onClick(ClickEvent event) { doSave(); } }); add(new ProjectDownloadPanel(getProjectKey().get(), true)); initDescription(); grid = new LabeledWidgetsGrid(); pluginOptionsPanel = new FlowPanel(); actionsGrid = new LabeledWidgetsGrid(); initProjectOptions(); initAgreements(); add(grid); add(pluginOptionsPanel); add(saveProject); add(actionsGrid);
4676 1242678012GerritCodeReview/gerritDave Borowitz0ec2763bc3e06c93eb02024003af95a2190ecbba None TODO Replace CreateChangeDialog with a nicer looking display. SATD_ADDED call(Button, String) static void call(Button b, final String project) // TODO Replace CreateChangeDialog with a nicer looking display. b.setEnabled(false); new CreateChangeDialog(b, new Project.NameKey(project)) { { sendButton.setText(Util.C.buttonCreate()); message.setText(Util.C.buttonCreateDescription()); } @Override public void onSend() { ChangeApi.createChange(project, getDestinationBranch(), message.getText(), new GerritCallback() { @Override public void onSuccess(ChangeInfo result) { sent = true; hide(); Gerrit.display(PageLinks.toChange(result.legacy_id())); } @Override public void onFailure(Throwable caught) { enableButtons(true); super.onFailure(caught); } }); } }.center();
4674 1242678010GerritCodeReview/gerritDavid Pursehouse9adf60e96fdce8c62cd33e0b6dd8fa6a45846932 None perhaps it's compressed SATD_ADDED start() public void start() throws UnableToCompleteException Server newServer = new Server(); ServerConnector connector = new ServerConnector(newServer); connector.setHost(bindAddress); connector.setPort(port); connector.setReuseAddress(false); connector.setSoLingerTime(0); newServer.addConnector(connector); ServletContextHandler handler = new ServletContextHandler(ServletContextHandler.SESSIONS); handler.setContextPath("/"); handler.addServlet(new ServletHolder(new HttpServlet() { @Override protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { handleRequest(request.getPathInfo(), request, response); } }), "/*"); handler.addFilter(GzipFilter.class, "/*", EnumSet.allOf(DispatcherType.class)); newServer.setHandler(handler); try { newServer.start(); } catch (Exception e) { logger.log(TreeLogger.ERROR, "cannot start web server", e); throw new UnableToCompleteException(); } this.server = newServer;
4672 1242678009GerritCodeReview/gerritDavid Pursehouseddce305901f716d4902cdb1a4a61db778930e1bc None TODO(davido): Consider to modify path column in edit mode as well to navigate to edit dialog instead of to SBS2. SATD_ADDED render(SafeHtmlBuilder, FileInfo) private void render(SafeHtmlBuilder sb, FileInfo info) sb.openTr(); sb.openTd().setStyleName(R.css().pointer()).closeTd(); // TODO(davido): Consider to modify path column in edit mode as well // to navigate to edit dialog instead of to SBS2. if (mode == Mode.REVIEW) { columnReviewed(sb, info); } else { columnEdit(sb, info); } columnStatus(sb, info); columnPath(sb, info); columnComments(sb, info); columnDelta1(sb, info); columnDelta2(sb, info); sb.closeTr();
4671 1242678008GerritCodeReview/gerritDavid Ostrovsky395255f026bddc58854ddb097a3e94d30120b5eb None Only expose publish action when the edit is on top of current ps SATD_ADDED ActionInfo> fillActions(ChangeEdit) private static Map fillActions(ChangeEdit edit) Map actions = Maps.newTreeMap(); UiAction.Description descr = new UiAction.Description(); PrivateInternals_UiActionDescription.setId(descr, "/"); PrivateInternals_UiActionDescription.setMethod(descr, "DELETE"); descr.setTitle("Delete edit"); actions.put(descr.getId(), new ActionInfo(descr)); // Only expose publish action when the edit is on top of current ps PatchSet.Id current = edit.getChange().currentPatchSetId(); PatchSet basePs = edit.getBasePatchSet(); if (basePs.getId().equals(current)) { descr = new UiAction.Description(); PrivateInternals_UiActionDescription.setId(descr, "publish"); PrivateInternals_UiActionDescription.setMethod(descr, "POST"); descr.setTitle("Publish edit"); actions.put(descr.getId(), new ActionInfo(descr)); } return actions;
4670 1242677902GerritCodeReview/gerritDavid Pursehouse22e43cc82355a3bef6e575539cda165aaec81b28 Email the reviewers The user knows they added themselves, don't bother emailing them. Email the reviewers The user knows they added themselves, don't bother emailing them. CLASS_OR_METHOD_CHANGED emailReviewers(Change, List) private void emailReviewers(Change change, List added) throws OrmException, EmailException if (added.isEmpty()) { return; } // Email the reviewers // // The user knows they added themselves, don't bother emailing them. List toMail = Lists.newArrayListWithCapacity(added.size()); for (PatchSetApproval psa : added) { if (!psa.getAccountId().equals(currentUser.getAccountId())) { toMail.add(psa.getAccountId()); } } if (!toMail.isEmpty()) { try { AddReviewerSender cm = addReviewerSenderFactory.create(change); cm.setFrom(currentUser.getAccountId()); cm.addReviewers(toMail); cm.send(); } catch (Exception err) { log.error("Cannot send email to new reviewers of change " + change.getId(), err); } }
4669 1242677837GerritCodeReview/gerritDavid Pursehousedf63f1aae5ff966a46a0b3382d5f1606411fd0f7 For a file with content merge conflict that we produced a result above on, collapse the file down to a single stage 0 with just the blob content, and a randomly selected mode (the lowest stage, which should be the merge base, or ours). For a file with content merge conflict that we produced a result above on, collapse the file down to a single stage 0 with just the blob content, and a randomly selected mode (the lowest stage, which should be the merge base, or ours). CLASS_OR_METHOD_CHANGED automerge(Repository, RevWalk, RevCommit, ThreeWayMergeStrategy, boolean) public static RevTree automerge(Repository repo, RevWalk rw, RevCommit b, ThreeWayMergeStrategy mergeStrategy, boolean save) throws IOException String hash = b.name(); String refName = RefNames.REFS_CACHE_AUTOMERGE + hash.substring(0, 2) + "/" + hash.substring(2); Ref ref = repo.getRef(refName); if (ref != null && ref.getObjectId() != null) { return rw.parseTree(ref.getObjectId()); } ResolveMerger m = (ResolveMerger) mergeStrategy.newMerger(repo, true); final ObjectInserter ins = repo.newObjectInserter(); try { DirCache dc = DirCache.newInCore(); m.setDirCache(dc); m.setObjectInserter(new ObjectInserter.Filter() { @Override protected ObjectInserter delegate() { return ins; } @Override public void flush() { } @Override public void release() { } }); boolean couldMerge; try { couldMerge = m.merge(b.getParents()); } catch (IOException e) { // It is not safe to continue further down in this method as throwing // an exception most likely means that the merge tree was not created // and m.getMergeResults() is empty. This would mean that all paths are // unmerged and Gerrit UI would show all paths in the patch list. log.warn("Error attempting automerge " + refName, e); return null; } ObjectId treeId; if (couldMerge) { treeId = m.getResultTreeId(); } else { RevCommit ours = b.getParent(0); RevCommit theirs = b.getParent(1); rw.parseBody(ours); rw.parseBody(theirs); String oursMsg = ours.getShortMessage(); String theirsMsg = theirs.getShortMessage(); String oursName = String.format("HEAD (%s %s)", ours.abbreviate(6).name(), oursMsg.substring(0, Math.min(oursMsg.length(), 60))); String theirsName = String.format("BRANCH (%s %s)", theirs.abbreviate(6).name(), theirsMsg.substring(0, Math.min(theirsMsg.length(), 60))); MergeFormatter fmt = new MergeFormatter(); Map> r = m.getMergeResults(); Map resolved = new HashMap<>(); for (Map.Entry> entry : r.entrySet()) { MergeResult p = entry.getValue(); TemporaryBuffer buf = new TemporaryBuffer.LocalFile(10 * 1024 * 1024); try { fmt.formatMerge(buf, p, "BASE", oursName, theirsName, "UTF-8"); buf.close(); InputStream in = buf.openInputStream(); try { resolved.put(entry.getKey(), ins.insert(Constants.OBJ_BLOB, buf.length(), in)); } finally { in.close(); } } finally { buf.destroy(); } } DirCacheBuilder builder = dc.builder(); int cnt = dc.getEntryCount(); for (int i = 0; i < cnt; ) { DirCacheEntry entry = dc.getEntry(i); if (entry.getStage() == 0) { builder.add(entry); i++; continue; } int next = dc.nextEntry(i); String path = entry.getPathString(); DirCacheEntry res = new DirCacheEntry(path); if (resolved.containsKey(path)) { // For a file with content merge conflict that we produced a result // above on, collapse the file down to a single stage 0 with just // the blob content, and a randomly selected mode (the lowest stage, // which should be the merge base, or ours). res.setFileMode(entry.getFileMode()); res.setObjectId(resolved.get(path)); } else if (next == i + 1) { // If there is exactly one stage present, shouldn't be a conflict... res.setFileMode(entry.getFileMode()); res.setObjectId(entry.getObjectId()); } else if (next == i + 2) { // Two stages suggests a delete/modify conflict. Pick the higher // stage as the automatic result. entry = dc.getEntry(i + 1); res.setFileMode(entry.getFileMode()); res.setObjectId(entry.getObjectId()); } else { // 3 stage conflict, no resolve above // Punt on the 3-stage conflict and show the base, for now. res.setFileMode(entry.getFileMode()); res.setObjectId(entry.getObjectId()); } builder.add(res); i = next; } builder.finish(); treeId = dc.writeTree(ins); } ins.flush(); if (save) { RefUpdate update = repo.updateRef(refName); update.setNewObjectId(treeId); update.disableRefLog(); update.forceUpdate(); } return rw.lookupTree(treeId); } finally { ins.release(); }
4668 1242678007GerritCodeReview/gerritDavid Ostrovsky8e75f5070e4eb638e5d9ecb6c016321b7c62c86d None Edit is created on top of current patch set by deleting path. Even if the latest patch set changed since the user triggered the operation, deleting the whole file is probably still what they intended. SATD_ADDED views() public DynamicMap> views() return views;
4667 1242678005GerritCodeReview/gerritYacob Yonasc93e841de36eacbed3813f18a998e8a98ab74af3 TODO(dborowitz): Smarter bucketing: pick a bucket start time T and include all events up to T + TS_WINDOW_MS but no further. Interleaving different authors complicates things. TODO(dborowitz): Smarter bucketing: pick a bucket start time T and include all events up to T + TS_WINDOW_MS but no further. Interleaving different authors complicates things. CLASS_OR_METHOD_CHANGED rebuildAsync(Change, ListeningExecutorService, BatchRefUpdate, BatchRefUpdate, Repository, Repository) public ListenableFuture rebuildAsync(final Change change, ListeningExecutorService executor, final BatchRefUpdate bru, final BatchRefUpdate bruForDrafts, final Repository changeRepo, final Repository allUsersRepo) return executor.submit(new Callable() { @Override public Void call() throws Exception { rebuild(change, bru, bruForDrafts, changeRepo, allUsersRepo); return null; } });
4666 1242678005GerritCodeReview/gerritShawn Pearce904c08183c645ed9644fd0babdb8e670f226a06b TODO(dborowitz): Smarter bucketing: pick a bucket start time T and include all events up to T + TS_WINDOW_MS but no further. Interleaving different authors complicates things. TODO(dborowitz): Smarter bucketing: pick a bucket start time T and include all events up to T + TS_WINDOW_MS but no further. Interleaving different authors complicates things. CLASS_OR_METHOD_CHANGED rebuildAsync(Change, ListeningExecutorService, BatchRefUpdate) public ListenableFuture rebuildAsync(final Change change, ListeningExecutorService executor, final BatchRefUpdate bru) return executor.submit(new Callable() { @Override public Void call() throws Exception { rebuild(change, bru); return null; } });
4665 1242678006GerritCodeReview/gerritDavid Ostrovsky14fcdd7458ea1c870b6fd2a3cedb9d57e1f26c8b None TODO(davido): Allow to resolve conflicts inline SATD_ADDED rebaseEdit(ChangeEdit, PatchSet) public RefUpdate.Result rebaseEdit(ChangeEdit edit, PatchSet current) throws AuthException, InvalidChangeOperationException, IOException if (!currentUser.get().isIdentifiedUser()) { throw new AuthException("Authentication required"); } Change change = edit.getChange(); IdentifiedUser me = (IdentifiedUser) currentUser.get(); String refName = editRefName(me.getAccountId(), change.getId()); Repository repo = gitManager.openRepository(change.getProject()); try { RevWalk rw = new RevWalk(repo); ObjectInserter inserter = repo.newObjectInserter(); try { RevCommit editCommit = edit.getEditCommit(); RevCommit mergeTip = rw.parseCommit(ObjectId.fromString(current.getRevision().get())); ThreeWayMerger m = MergeStrategy.RESOLVE.newMerger(repo, true); m.setObjectInserter(inserter); m.setBase(editCommit.getParent(0)); if (m.merge(mergeTip, editCommit)) { ObjectId tree = m.getResultTreeId(); CommitBuilder mergeCommit = new CommitBuilder(); mergeCommit.setTreeId(tree); mergeCommit.setParentId(mergeTip); mergeCommit.setAuthor(editCommit.getAuthorIdent()); mergeCommit.setCommitter(new PersonIdent(editCommit.getCommitterIdent(), TimeUtil.nowTs())); mergeCommit.setMessage(editCommit.getFullMessage()); ObjectId newEdit = inserter.insert(mergeCommit); inserter.flush(); return update(repo, me, refName, rw, editCommit, newEdit); } else { // TODO(davido): Allow to resolve conflicts inline throw new InvalidChangeOperationException("merge conflict"); } } finally { rw.release(); inserter.release(); } } finally { repo.close(); }
4664 1242678005GerritCodeReview/gerritYacob Yonas3f4a1748b22003539e797d7ed36dc41535e1b0b2 None TODO(dborowitz): Smarter bucketing: pick a bucket start time T and include all events up to T + TS_WINDOW_MS but no further. Interleaving different authors complicates things. SATD_ADDED rebuildAsync(Change, ListeningExecutorService) public ListenableFuture rebuildAsync(final Change change, ListeningExecutorService executor) return executor.submit(new Callable() { @Override public Void call() throws Exception { rebuild(change, null); return null; } });
4662 1242678004GerritCodeReview/gerritDavid Ostrovskya570dbfb06570696f91d9136ee43d9a26ce339e7 None TODO(davido): This should happen in the same BatchRefUpdate. SATD_ADDED publish(ChangeEdit) public void publish(ChangeEdit edit) throws AuthException, NoSuchChangeException, IOException, InvalidChangeOperationException, OrmException, ResourceConflictException Change change = edit.getChange(); Repository repo = gitManager.openRepository(change.getProject()); try { RevWalk rw = new RevWalk(repo); ObjectInserter inserter = repo.newObjectInserter(); try { RevCommit editCommit = rw.parseCommit(edit.getRef().getObjectId()); if (editCommit == null) { throw new NoSuchChangeException(change.getId()); } PatchSet basePatchSet = getBasePatchSet(edit, editCommit); if (!basePatchSet.getId().equals(change.currentPatchSetId())) { throw new ResourceConflictException("only edit for current patch set can be published"); } insertPatchSet(edit, change, repo, rw, basePatchSet, squashEdit(repo, rw, inserter, editCommit, basePatchSet)); } finally { inserter.release(); rw.release(); } // TODO(davido): This should happen in the same BatchRefUpdate. deleteRef(repo, edit); } finally { repo.close(); }
4659 1242678003GerritCodeReview/gerritDave Borowitz638e70d22d52441880d9c57c3e10144abf03c0ef None TODO(dborowitz): Fix bug so this is included. SATD_ADDED allRefsVisibleWithRefsMetaConfig() public void allRefsVisibleWithRefsMetaConfig() throws Exception allow(Permission.READ, REGISTERED_USERS, "refs/*"); allow(Permission.READ, REGISTERED_USERS, "refs/meta/config"); assertRefs("HEAD", "refs/changes/01/1/1", "refs/changes/02/2/1", "refs/heads/branch", "refs/heads/master"); // TODO(dborowitz): Fix bug so this is included. // "refs/meta/config");
4658 1242678002GerritCodeReview/gerritDave Borowitz80915206e74d95e9eb7ce35059120b70f380ffbe None TODO(dborowitz): This should not be allowed. SATD_ADDED canReadAfterRollbackWithAllRefsVisible() public void canReadAfterRollbackWithAllRefsVisible() throws Exception allow(project, READ, DEVS, "refs/*"); RevCommit parent1 = repo.commit().create(); ObjectId id1 = repo.branch("branch1").commit().parent(parent1).create(); ProjectControl pc = util.user(project, DEVS); RevWalk rw = repo.getRevWalk(); assertTrue(pc.canReadCommit(rw, rw.parseCommit(parent1))); assertTrue(pc.canReadCommit(rw, rw.parseCommit(id1))); repo.branch("branch1").update(parent1); assertTrue(pc.canReadCommit(rw, rw.parseCommit(parent1))); // TODO(dborowitz): This should not be allowed. assertTrue(pc.canReadCommit(rw, rw.parseCommit(id1)));
4656 1242678000GerritCodeReview/gerritEdwin Kempin1ef3b37c887de32a238f321c59c4eb8924a1c4b9 None TODO(yyonas): Delete drafts if they already existed. SATD_ADDED addPublishedComments(ReviewDb, ChangeUpdate, Iterable) public void addPublishedComments(ReviewDb db, ChangeUpdate update, Iterable comments) throws OrmException for (PatchLineComment c : comments) { update.upsertComment(c); } db.patchComments().upsert(comments);
4655 1242677999GerritCodeReview/gerritYacob Yonas2ef433bc92c78511fca52d9374cd2030aa272c14 TODO(dborowitz): Use a ThreadLocal or use Joda. TODO(dborowitz): Use a ThreadLocal or use Joda. CLASS_OR_METHOD_CHANGED parseCommentsFromNotes(Repository, String, RevWalk, Change.Id, Multimap, Multimap, Status) public static NoteMap parseCommentsFromNotes(Repository repo, String refName, RevWalk walk, Change.Id changeId, Multimap commentsForBase, Multimap commentsForPs, Status status) throws IOException, ConfigInvalidException Ref ref = repo.getRef(refName); if (ref == null) { return null; } RevCommit commit = walk.parseCommit(ref.getObjectId()); NoteMap noteMap = NoteMap.read(walk.getObjectReader(), commit); for (Note note : noteMap) { byte[] bytes = walk.getObjectReader().open(note.getData(), Constants.OBJ_BLOB).getBytes(); List result = parseNote(bytes, changeId, status); if ((result == null) || (result.isEmpty())) { continue; } PatchSet.Id psId = result.get(0).getKey().getParentKey().getParentKey(); short side = result.get(0).getSide(); if (side == 0) { commentsForBase.putAll(psId, result); } else { commentsForPs.putAll(psId, result); } } return noteMap;
4653 1242677998GerritCodeReview/gerritLuca Milanesio195b9d7ece63e385192d22061e2d70130b3a90f9 None TODO(dborowitz): Use a ThreadLocal or use Joda. SATD_ADDED parseNote(byte[], Change.Id) public static List parseNote(byte[] note, Change.Id changeId) throws ConfigInvalidException List result = Lists.newArrayList(); int sizeOfNote = note.length; Charset enc = RawParseUtils.parseEncoding(note); MutableInteger curr = new MutableInteger(); curr.value = 0; boolean isForBase = (RawParseUtils.match(note, curr.value, PATCH_SET.getBytes(UTF_8))) < 0; PatchSet.Id psId = parsePsId(note, curr, changeId, enc, isForBase ? BASE_PATCH_SET : PATCH_SET); RevId revId = new RevId(parseStringField(note, curr, changeId, enc, REVISION)); PatchLineComment c = null; while (curr.value < sizeOfNote) { String previousFileName = c == null ? null : c.getKey().getParentKey().getFileName(); c = parseComment(note, curr, previousFileName, psId, revId, isForBase, enc); result.add(c); } return result;
4652 1242677997GerritCodeReview/gerritLuca Milanesio195b9d7ece63e385192d22061e2d70130b3a90f9 None TODO(yyonas): figure out how to handle this exception SATD_ADDED onLoad() protected void onLoad() throws IOException, ConfigInvalidException ObjectId rev = getRevision(); if (rev == null) { loadDefaults(); return; } RevWalk walk = new RevWalk(reader); try { Parser parser = new Parser(change, rev, walk, repoManager); parser.parseAll(); if (parser.status != null) { change.setStatus(parser.status); } approvals = parser.buildApprovals(); changeMessages = parser.buildMessages(); commentsForBase = ImmutableListMultimap.copyOf(parser.commentsForBase); commentsForPS = ImmutableListMultimap.copyOf(parser.commentsForPs); noteMap = parser.commentNoteMap; ImmutableSetMultimap.Builder reviewers = ImmutableSetMultimap.builder(); for (Map.Entry e : parser.reviewers.entrySet()) { reviewers.put(e.getValue(), e.getKey()); } this.reviewers = reviewers.build(); submitRecords = ImmutableList.copyOf(parser.submitRecords); } catch (ParseException e1) { // TODO(yyonas): figure out how to handle this exception throw new IOException(e1); } finally { walk.release(); }
4651 1242677996GerritCodeReview/gerritDave Borowitz086006ec35ef2a5bddb14a3a4ef427ad6085b01e None TODO(yyonas): atomic change is not propagated. SATD_ADDED abandonOneChange(Change) private void abandonOneChange(Change change) throws OrmException, NoSuchChangeException, IOException db.changes().beginTransaction(change.getId()); // TODO(dborowitz): support InternalUser in ChangeUpdate ChangeControl control = changeControlFactory.controlFor(change, identifiedUserFactory.create(change.getOwner())); ChangeUpdate update = updateFactory.create(control); try { change = db.changes().atomicUpdate(change.getId(), new AtomicUpdate() { @Override public Change update(Change change) { if (change.getStatus().isOpen()) { change.setStatus(Change.Status.ABANDONED); return change; } return null; } }); if (change != null) { ChangeMessage msg = new ChangeMessage(new ChangeMessage.Key(change.getId(), ChangeUtil.messageUUID(db)), null, change.getLastUpdatedOn(), change.currentPatchSetId()); msg.setMessage("Project was deleted."); // TODO(yyonas): atomic change is not propagated. cmUtil.addChangeMessage(db, update, msg); db.commit(); indexer.index(db, change); } } finally { db.rollback(); } update.commit();
4650 1242677995GerritCodeReview/gerritDave Borowitz086006ec35ef2a5bddb14a3a4ef427ad6085b01e None TODO(dborowitz): support InternalUser in ChangeUpdate SATD_ADDED abandonOneChange(Change) private void abandonOneChange(Change change) throws OrmException, NoSuchChangeException, IOException db.changes().beginTransaction(change.getId()); // TODO(dborowitz): support InternalUser in ChangeUpdate ChangeControl control = changeControlFactory.controlFor(change, identifiedUserFactory.create(change.getOwner())); ChangeUpdate update = updateFactory.create(control); try { change = db.changes().atomicUpdate(change.getId(), new AtomicUpdate() { @Override public Change update(Change change) { if (change.getStatus().isOpen()) { change.setStatus(Change.Status.ABANDONED); return change; } return null; } }); if (change != null) { ChangeMessage msg = new ChangeMessage(new ChangeMessage.Key(change.getId(), ChangeUtil.messageUUID(db)), null, change.getLastUpdatedOn(), change.currentPatchSetId()); msg.setMessage("Project was deleted."); // TODO(yyonas): atomic change is not propagated. cmUtil.addChangeMessage(db, update, msg); db.commit(); indexer.index(db, change); } } finally { db.rollback(); } update.commit();
4649 1242677994GerritCodeReview/gerritDave Borowitz086006ec35ef2a5bddb14a3a4ef427ad6085b01e None TODO(yyonas): atomic change is not propagated. SATD_ADDED sendMergeFail(ChangeNotes, ChangeMessage, boolean) private void sendMergeFail(ChangeNotes notes, final ChangeMessage msg, boolean makeNew) throws NoSuchChangeException, IOException PatchSetApproval submitter = null; try { submitter = approvalsUtil.getSubmitter(db, notes, notes.getChange().currentPatchSetId()); } catch (Exception e) { log.error("Cannot get submitter", e); } if (!makeNew) { RetryStatus retryStatus = getRetryStatus(submitter, msg, notes); if (retryStatus == RetryStatus.RETRY_NO_MESSAGE) { return; } else if (retryStatus == RetryStatus.UNSUBMIT) { makeNew = true; } } final boolean setStatusNew = makeNew; final Change c = notes.getChange(); Change change = null; ChangeUpdate update = null; try { db.changes().beginTransaction(c.getId()); try { change = db.changes().atomicUpdate(c.getId(), new AtomicUpdate() { @Override public Change update(Change c) { if (c.getStatus().isOpen()) { if (setStatusNew) { c.setStatus(Change.Status.NEW); } ChangeUtil.updated(c); } return c; } }); ChangeControl control = changeControl(change); // TODO(yyonas): atomic change is not propagated. update = updateFactory.create(control, c.getLastUpdatedOn()); cmUtil.addChangeMessage(db, update, msg); db.commit(); } finally { db.rollback(); } } catch (OrmException err) { log.warn("Cannot record merge failure message", err); } if (update != null) { update.commit(); } CheckedFuture indexFuture; if (change != null) { indexFuture = indexer.indexAsync(change.getId()); } else { indexFuture = null; } final PatchSetApproval from = submitter; workQueue.getDefaultQueue().submit(requestScopePropagator.wrap(new Runnable() { @Override public void run() { PatchSet patchSet; try { ReviewDb reviewDb = schemaFactory.open(); try { patchSet = reviewDb.patchSets().get(c.currentPatchSetId()); } finally { reviewDb.close(); } } catch (Exception e) { log.error("Cannot send email notifications about merge failure", e); return; } try { final MergeFailSender cm = mergeFailSenderFactory.create(c); if (from != null) { cm.setFrom(from.getAccountId()); } cm.setPatchSet(patchSet); cm.setChangeMessage(msg); cm.send(); } catch (Exception e) { log.error("Cannot send email notifications about merge failure", e); } } @Override public String toString() { return "send-email merge-failed"; } })); if (submitter != null) { try { hooks.doMergeFailedHook(c, accountCache.get(submitter.getAccountId()).getAccount(), db.patchSets().get(c.currentPatchSetId()), msg.getMessage(), db); } catch (OrmException ex) { log.error("Cannot run hook for merge failed " + c.getId(), ex); } } if (indexFuture != null) { try { indexFuture.checkedGet(); } catch (IOException e) { log.error("Failed to index new change message", e); } }
4648 1242677993GerritCodeReview/gerritDave Borowitz086006ec35ef2a5bddb14a3a4ef427ad6085b01e None TODO(yyonas): atomic update was not propagated SATD_ADDED apply(ChangeResource, Input) public Response apply(ChangeResource req, Input input) throws AuthException, OrmException, IOException if (input == null) { input = new Input(); } ChangeControl control = req.getControl(); Change change = req.getChange(); if (!control.canEditTopicName()) { throw new AuthException("changing topic not permitted"); } ReviewDb db = dbProvider.get(); final String newTopicName = Strings.nullToEmpty(input.topic); String oldTopicName = Strings.nullToEmpty(change.getTopic()); if (!oldTopicName.equals(newTopicName)) { String summary; if (oldTopicName.isEmpty()) { summary = "Topic set to " + newTopicName; } else if (newTopicName.isEmpty()) { summary = "Topic " + oldTopicName + " removed"; } else { summary = String.format("Topic changed from %s to %s", oldTopicName, newTopicName); } IdentifiedUser currentUser = ((IdentifiedUser) control.getCurrentUser()); ChangeMessage cmsg = new ChangeMessage(new ChangeMessage.Key(change.getId(), ChangeUtil.messageUUID(db)), currentUser.getAccountId(), TimeUtil.nowTs(), change.currentPatchSetId()); cmsg.setMessage(summary); ChangeUpdate update; db.changes().beginTransaction(change.getId()); try { change = db.changes().atomicUpdate(change.getId(), new AtomicUpdate() { @Override public Change update(Change change) { change.setTopic(Strings.emptyToNull(newTopicName)); ChangeUtil.updated(change); return change; } }); // TODO(yyonas): atomic update was not propagated update = updateFactory.create(control); cmUtil.addChangeMessage(db, update, cmsg); db.commit(); } finally { db.rollback(); } update.commit(); CheckedFuture indexFuture = indexer.indexAsync(change.getId()); hooks.doTopicChangedHook(change, currentUser.getAccount(), oldTopicName, db); indexFuture.checkedGet(); } return Strings.isNullOrEmpty(newTopicName) ? Response.none() : Response.ok(newTopicName);
4647 1242677992GerritCodeReview/gerritDave Borowitz086006ec35ef2a5bddb14a3a4ef427ad6085b01e None TODO(yyonas): atomic update was not propagated SATD_ADDED apply(ChangeResource, RestoreInput) public ChangeInfo apply(ChangeResource req, RestoreInput input) throws AuthException, ResourceConflictException, OrmException, IOException ChangeControl control = req.getControl(); IdentifiedUser caller = (IdentifiedUser) control.getCurrentUser(); Change change = req.getChange(); if (!control.canRestore()) { throw new AuthException("restore not permitted"); } else if (change.getStatus() != Status.ABANDONED) { throw new ResourceConflictException("change is " + status(change)); } ChangeMessage message; ChangeUpdate update; ReviewDb db = dbProvider.get(); db.changes().beginTransaction(change.getId()); try { change = db.changes().atomicUpdate(change.getId(), new AtomicUpdate() { @Override public Change update(Change change) { if (change.getStatus() == Status.ABANDONED) { change.setStatus(Status.NEW); ChangeUtil.updated(change); return change; } return null; } }); if (change == null) { throw new ResourceConflictException("change is " + status(db.changes().get(req.getChange().getId()))); } // TODO(yyonas): atomic update was not propagated update = updateFactory.create(control); message = newMessage(input, caller, change); cmUtil.addChangeMessage(db, update, message); db.commit(); } finally { db.rollback(); } update.commit(); CheckedFuture f = mergeabilityChecker.newCheck().addChange(change).reindex().runAsync(); try { ReplyToChangeSender cm = restoredSenderFactory.create(change); cm.setFrom(caller.getAccountId()); cm.setChangeMessage(message); cm.send(); } catch (Exception e) { log.error("Cannot email update for change " + change.getChangeId(), e); } hooks.doChangeRestoredHook(change, caller.getAccount(), db.patchSets().get(change.currentPatchSetId()), Strings.emptyToNull(input.message), dbProvider.get()); ChangeInfo result = json.format(change); f.checkedGet(); return result;
4646 1242677991GerritCodeReview/gerritDave Borowitz086006ec35ef2a5bddb14a3a4ef427ad6085b01e None TODO(yyonas): atomic update was not propagated SATD_ADDED apply(ChangeResource, AbandonInput) public ChangeInfo apply(ChangeResource req, AbandonInput input) throws AuthException, ResourceConflictException, OrmException, IOException ChangeControl control = req.getControl(); IdentifiedUser caller = (IdentifiedUser) control.getCurrentUser(); Change change = req.getChange(); if (!control.canAbandon()) { throw new AuthException("abandon not permitted"); } else if (!change.getStatus().isOpen()) { throw new ResourceConflictException("change is " + status(change)); } else if (change.getStatus() == Change.Status.DRAFT) { throw new ResourceConflictException("draft changes cannot be abandoned"); } ChangeMessage message; ChangeUpdate update; ReviewDb db = dbProvider.get(); db.changes().beginTransaction(change.getId()); try { change = db.changes().atomicUpdate(change.getId(), new AtomicUpdate() { @Override public Change update(Change change) { if (change.getStatus().isOpen()) { change.setStatus(Change.Status.ABANDONED); ChangeUtil.updated(change); return change; } return null; } }); if (change == null) { throw new ResourceConflictException("change is " + status(db.changes().get(req.getChange().getId()))); } // TODO(yyonas): atomic update was not propagated update = updateFactory.create(control, change.getLastUpdatedOn()); message = newMessage(input, caller, change); cmUtil.addChangeMessage(db, update, message); db.commit(); } finally { db.rollback(); } update.commit(); CheckedFuture indexFuture = indexer.indexAsync(change.getId()); try { ReplyToChangeSender cm = abandonedSenderFactory.create(change); cm.setFrom(caller.getAccountId()); cm.setChangeMessage(message); cm.send(); } catch (Exception e) { log.error("Cannot email update for change " + change.getChangeId(), e); } hooks.doChangeAbandonedHook(change, caller.getAccount(), db.patchSets().get(change.currentPatchSetId()), Strings.emptyToNull(input.message), db); ChangeInfo result = json.format(change); indexFuture.checkedGet(); return result;
4645 1242677990GerritCodeReview/gerritDavid Pursehouse58ba20634fd1cb43a435153dd117119d8adbbd87 None TODO - dborowitz: add NEW_CHANGE type for default. SATD_ADDED getChangeKindInternal(ChangeKindCache, ReviewDb, Change, PatchSet, ChangeData.Factory, ProjectCache, GitRepositoryManager) private static ChangeKind getChangeKindInternal(ChangeKindCache cache, ReviewDb db, Change change, PatchSet patch, ChangeData.Factory changeDataFactory, ProjectCache projectCache, GitRepositoryManager repoManager) Repository repo = null; // TODO - dborowitz: add NEW_CHANGE type for default. ChangeKind kind = ChangeKind.REWORK; // Trivial case: if we're on the first patch, we don't need to open // the repository. if (patch.getId().get() > 1) { try { ProjectState projectState = projectCache.checkedGet(change.getProject()); repo = repoManager.openRepository(change.getProject()); ChangeData cd = changeDataFactory.create(db, change); Collection patchSetCollection = cd.patches(); PatchSet priorPs = patch; for (PatchSet ps : patchSetCollection) { if (ps.getId().get() < patch.getId().get() && (ps.getId().get() > priorPs.getId().get() || priorPs == patch)) { // We only want the previous patch set, so walk until the last one priorPs = ps; } } // If we still think the previous patch is the current patch, // we only have one patch set. Return the default. // This can happen if a user creates a draft, uploads a second patch, // and deletes the draft. if (priorPs != patch) { kind = cache.getChangeKind(projectState, repo, ObjectId.fromString(priorPs.getRevision().get()), ObjectId.fromString(patch.getRevision().get())); } } catch (IOException | OrmException e) { // Do nothing; assume we have a complex change log.warn("Unable to get change kind for patchSet " + patch.getPatchSetId() + "of change " + change.getChangeId(), e); } finally { if (repo != null) { repo.close(); } } } return kind;
4643 1242677988GerritCodeReview/gerritEdwin Kempinf712cdb27c74d8e19f19e3c15c9bbc32069187d2 TODO Kill /accounts/*/avatar URL. TODO Kill /accounts/*/avatar URL. CLASS_OR_METHOD_CHANGED setAccount(AccountInfo, int, boolean) public void setAccount(AccountInfo account, int size, boolean addPopup) if (account == null) { setVisible(false); } else if (isGerritServer(account)) { setVisible(true); setResource(Gerrit.RESOURCES.gerritAvatar26()); } else if (account.has_avatar_info()) { setVisible(false); AvatarInfo info = account.avatar(size); if (info != null) { setWidth(info.width() > 0 ? info.width() + "px" : ""); setHeight(info.height() > 0 ? info.height() + "px" : ""); setUrl(info.url()); popup(account, addPopup); } else if (account.email() != null) { loadAvatar(account, size, addPopup); } } else if (account.email() != null) { loadAvatar(account, size, addPopup); } else { setVisible(false); }
4641 1242677987GerritCodeReview/gerritDave Borowitza19b319af18e912d85964b7b989a84b1b078169b None We can only approximately reconstruct what the submit rule evaluator would have done. These should really come from a stored submit record. SATD_ADDED iterator() public Iterator iterator() return toList().iterator();
4640 1242677809GerritCodeReview/gerritDave Borowitz4c27b10dc5def4aeac2fd5560a00478acc456db9 If two or more projects contains \"query\" as substring create an OrPredicate holding predicates for all these projects, otherwise if only one contains that, return only that one predicate by itself. Skip. SATD_REMOVED defaultField(String) protected Predicate defaultField(String query) if (query.startsWith("refs/")) { return ref(query); } else if (DEF_CHANGE.matcher(query).matches()) { return change(query); } List> predicates = Lists.newArrayListWithCapacity(9); try { predicates.add(commit(query)); } catch (IllegalArgumentException e) { // Skip. } try { predicates.add(owner(query)); } catch (OrmException | QueryParseException e) { // Skip. } try { predicates.add(reviewer(query)); } catch (OrmException | QueryParseException e) { // Skip. } try { predicates.add(file(query)); } catch (QueryParseException e) { // Skip. } try { predicates.add(label(query)); } catch (OrmException | QueryParseException e) { // Skip. } try { predicates.add(message(query)); } catch (QueryParseException e) { // Skip. } try { predicates.add(comment(query)); } catch (QueryParseException e) { // Skip. } try { predicates.add(projects(query)); } catch (QueryParseException e) { // Skip. } predicates.add(ref(query)); predicates.add(branch(query)); predicates.add(topic(query)); return Predicate.or(predicates);
4639 1242677883GerritCodeReview/gerritDavid Pursehouse3a20cefc4bbd025173f83489855bbf9348bea642 If no existing label is being set to 0, hack in the caller as a reviewer by picking the first server-wide LabelType. If no existing label is being set to 0, hack in the caller as a reviewer by picking the first server-wide LabelType. CLASS_OR_METHOD_CHANGED forceCallerAsReviewer(RevisionResource, Map, List, List) private void forceCallerAsReviewer(RevisionResource rsrc, Map current, List ups, List del) if (current.isEmpty() && ups.isEmpty()) { // TODO Find another way to link reviewers to changes. if (del.isEmpty()) { // If no existing label is being set to 0, hack in the caller // as a reviewer by picking the first server-wide LabelType. PatchSetApproval c = new PatchSetApproval(new PatchSetApproval.Key(rsrc.getPatchSet().getId(), rsrc.getAccountId(), rsrc.getControl().getLabelTypes().getLabelTypes().get(0).getLabelId()), (short) 0, TimeUtil.nowTs()); c.setGranted(timestamp); c.cache(change); ups.add(c); } else { // Pick a random label that is about to be deleted and keep it. Iterator i = del.iterator(); PatchSetApproval c = i.next(); c.setValue((short) 0); c.setGranted(timestamp); c.cache(change); i.remove(); ups.add(c); } }
4638 1242677882GerritCodeReview/gerritDavid Pursehouse3a20cefc4bbd025173f83489855bbf9348bea642 TODO Find another way to link reviewers to changes. TODO Find another way to link reviewers to changes. CLASS_OR_METHOD_CHANGED forceCallerAsReviewer(RevisionResource, Map, List, List) private void forceCallerAsReviewer(RevisionResource rsrc, Map current, List ups, List del) if (current.isEmpty() && ups.isEmpty()) { // TODO Find another way to link reviewers to changes. if (del.isEmpty()) { // If no existing label is being set to 0, hack in the caller // as a reviewer by picking the first server-wide LabelType. PatchSetApproval c = new PatchSetApproval(new PatchSetApproval.Key(rsrc.getPatchSet().getId(), rsrc.getAccountId(), rsrc.getControl().getLabelTypes().getLabelTypes().get(0).getLabelId()), (short) 0, TimeUtil.nowTs()); c.setGranted(timestamp); c.cache(change); ups.add(c); } else { // Pick a random label that is about to be deleted and keep it. Iterator i = del.iterator(); PatchSetApproval c = i.next(); c.setValue((short) 0); c.setGranted(timestamp); c.cache(change); i.remove(); ups.add(c); } }
4635 1242677855GerritCodeReview/gerritDavid Ostrovsky520f8a9e23b36194192dc15a11d0138b00d787ba javac generics bug javac generics bug FILE_PATH_CHANGED keys(JavaScriptObject) public static Set keys(JavaScriptObject obj) if (obj != null) { return new JSONObject(obj).keySet(); } return Collections.emptySet();
4634 1242677892GerritCodeReview/gerritDave Borowitze342270dba73886f6c3042c821ebc3fac514e5a9 Might be a group name, be nice and accept unique names. Might be a group name, be nice and accept unique names. CLASS_OR_METHOD_CHANGED parseId(String) public GroupDescription.Basic parseId(String id) AccountGroup.UUID uuid = new AccountGroup.UUID(id); if (groupBackend.handles(uuid)) { GroupDescription.Basic d = groupBackend.get(uuid); if (d != null) { return d; } } // Might be a legacy AccountGroup.Id. if (id.matches("^[1-9][0-9]*$")) { try { AccountGroup.Id legacyId = AccountGroup.Id.parse(id); return groupControlFactory.controlFor(legacyId).getGroup(); } catch (IllegalArgumentException invalidId) { } catch (NoSuchGroupException e) { } } // Might be a group name, be nice and accept unique names. GroupReference ref = GroupBackends.findExactSuggestion(groupBackend, id); if (ref != null) { GroupDescription.Basic d = groupBackend.get(ref.getUUID()); if (d != null) { return d; } } return null;
4633 1242677986GerritCodeReview/gerritDavid Ostrovskyf067fc081ac2a14c0563a332b4f289f51c0ac6d2 None TODO(davido): find a better way to enforce cache invalidation. SATD_ADDED deleteOrUpdateDraftChange(PatchSet.Id, Change) private void deleteOrUpdateDraftChange(PatchSet.Id patchSetId, Change change) throws OrmException, ResourceNotFoundException, IOException if (dbProvider.get().patchSets().byChange(change.getId()).toList().size() == 0) { deleteDraftChange(patchSetId); } else { if (change.currentPatchSetId().equals(patchSetId)) { updateChange(dbProvider.get(), change, previousPatchSetInfo(patchSetId)); } else { // TODO(davido): find a better way to enforce cache invalidation. updateChange(dbProvider.get(), change, null); } }
4632 1242677883GerritCodeReview/gerritDave Borowitzd0624718cb405cbbf1a49323a2e4eb48045fc6a8 If no existing label is being set to 0, hack in the caller as a reviewer by picking the first server-wide LabelType. If no existing label is being set to 0, hack in the caller as a reviewer by picking the first server-wide LabelType. CLASS_OR_METHOD_CHANGED forceCallerAsReviewer(RevisionResource, Map, List, List) private void forceCallerAsReviewer(RevisionResource rsrc, Map current, List ups, List del) if (current.isEmpty() && ups.isEmpty()) { // TODO Find another way to link reviewers to changes. if (del.isEmpty()) { // If no existing label is being set to 0, hack in the caller // as a reviewer by picking the first server-wide LabelType. PatchSetApproval c = new PatchSetApproval(new PatchSetApproval.Key(rsrc.getPatchSet().getId(), rsrc.getAccountId(), rsrc.getControl().getLabelTypes().getLabelTypes().get(0).getLabelId()), (short) 0, TimeUtil.nowTs()); c.setGranted(timestamp); ups.add(c); } else { // Pick a random label that is about to be deleted and keep it. Iterator i = del.iterator(); PatchSetApproval c = i.next(); c.setValue((short) 0); c.setGranted(timestamp); i.remove(); ups.add(c); } }
4631 1242677882GerritCodeReview/gerritDave Borowitzd0624718cb405cbbf1a49323a2e4eb48045fc6a8 TODO Find another way to link reviewers to changes. TODO Find another way to link reviewers to changes. CLASS_OR_METHOD_CHANGED forceCallerAsReviewer(RevisionResource, Map, List, List) private void forceCallerAsReviewer(RevisionResource rsrc, Map current, List ups, List del) if (current.isEmpty() && ups.isEmpty()) { // TODO Find another way to link reviewers to changes. if (del.isEmpty()) { // If no existing label is being set to 0, hack in the caller // as a reviewer by picking the first server-wide LabelType. PatchSetApproval c = new PatchSetApproval(new PatchSetApproval.Key(rsrc.getPatchSet().getId(), rsrc.getAccountId(), rsrc.getControl().getLabelTypes().getLabelTypes().get(0).getLabelId()), (short) 0, TimeUtil.nowTs()); c.setGranted(timestamp); ups.add(c); } else { // Pick a random label that is about to be deleted and keep it. Iterator i = del.iterator(); PatchSetApproval c = i.next(); c.setValue((short) 0); c.setGranted(timestamp); i.remove(); ups.add(c); } }
4630 1242677986GerritCodeReview/gerritDavid Pursehousea50fa6761f306aa774e34dd5dc356180743dcbbd None TODO(davido): find a better way to enforce cache invalidation. SATD_ADDED deleteOrUpdateDraftChange(PatchSet.Id, Change) private void deleteOrUpdateDraftChange(PatchSet.Id patchSetId, Change change) throws OrmException, ResourceNotFoundException, IOException if (dbProvider.get().patchSets().byChange(change.getId()).toList().size() == 0) { deleteDraftChange(patchSetId); } else { if (change.currentPatchSetId().equals(patchSetId)) { updateChange(dbProvider.get(), change, previousPatchSetInfo(patchSetId)); } else { // TODO(davido): find a better way to enforce cache invalidation. updateChange(dbProvider.get(), change, null); } }
4628 1242677985GerritCodeReview/gerritDavid Ostrovskya225e3183b99711e1843cf8a0148ca1da88a6edf None Don't use asserts from PushOneCommit so we can test the round-trip through JSON instead of querying the DB directly. SATD_ADDED assertApproval(PushOneCommit.Result, int) private void assertApproval(PushOneCommit.Result r, int expected) throws Exception // Don't use asserts from PushOneCommit so we can test the round-trip // through JSON instead of querying the DB directly. LabelInfo cr = getChange(r).labels.get("Code-Review"); assertEquals(1, cr.all.size()); assertEquals("User", cr.all.get(0).name); assertEquals(expected, cr.all.get(0).value.intValue());
4627 1242677984GerritCodeReview/gerritDave Borowitzd064abea4f107cdb4293ca0fccc02c5ee3fc4d0b None TODO(dborowitz): Store normalized labels in notedb. SATD_ADDED saveApprovals(Change, ChangeNotes, PatchSet.Id) private PatchSetApproval saveApprovals(Change c, ChangeNotes notes, PatchSet.Id merged) throws OrmException // Flatten out existing approvals for this patch set based upon the current // permissions. Once the change is closed the approvals are not updated at // presentation view time, except for zero votes used to indicate a reviewer // was added. So we need to make sure votes are accurate now. This way if // permissions get modified in the future, historical records stay accurate. PatchSetApproval submitter = null; try { List approvals = approvalsUtil.byPatchSet(db, notes, merged); Set toDelete = Sets.newHashSetWithExpectedSize(approvals.size()); for (PatchSetApproval a : approvals) { if (a.getValue() != 0) { toDelete.add(a.getKey()); } } approvals = labelNormalizer.normalize(c, approvals); for (PatchSetApproval a : approvals) { toDelete.remove(a.getKey()); if (a.getValue() > 0 && a.isSubmit()) { if (submitter == null || a.getGranted().compareTo(submitter.getGranted()) > 0) { submitter = a; } } } // TODO(dborowitz): Store normalized labels in notedb. db.patchSetApprovals().update(approvals); db.patchSetApprovals().deleteKeys(toDelete); } catch (NoSuchChangeException err) { throw new OrmException(err); } return submitter;
4626 1242677881GerritCodeReview/gerritDave Borowitzd064abea4f107cdb4293ca0fccc02c5ee3fc4d0b TODO Allow updating some labels even when closed. TODO Allow updating some labels even when closed. CLASS_OR_METHOD_CHANGED updateLabels(RevisionResource, ChangeUpdate, Map) private boolean updateLabels(RevisionResource rsrc, ChangeUpdate update, Map labels) throws OrmException if (labels == null) { labels = Collections.emptyMap(); } List del = Lists.newArrayList(); List ins = Lists.newArrayList(); List upd = Lists.newArrayList(); Map current = scanLabels(rsrc, del); LabelTypes labelTypes = rsrc.getControl().getLabelTypes(); for (Map.Entry ent : labels.entrySet()) { String name = ent.getKey(); LabelType lt = checkNotNull(labelTypes.byLabel(name), name); if (change.getStatus().isClosed()) { // TODO Allow updating some labels even when closed. continue; } PatchSetApproval c = current.remove(name); String normName = lt.getName(); if (ent.getValue() == null || ent.getValue() == 0) { // User requested delete of this label. if (c != null) { if (c.getValue() != 0) { addLabelDelta(normName, (short) 0); } del.add(c); update.putApproval(ent.getKey(), (short) 0); } } else if (c != null && c.getValue() != ent.getValue()) { c.setValue(ent.getValue()); c.setGranted(timestamp); upd.add(c); addLabelDelta(normName, c.getValue()); categories.put(normName, c.getValue()); update.putApproval(ent.getKey(), ent.getValue()); } else if (c != null && c.getValue() == ent.getValue()) { current.put(normName, c); } else if (c == null) { c = new PatchSetApproval(new PatchSetApproval.Key(rsrc.getPatchSet().getId(), rsrc.getAccountId(), lt.getLabelId()), ent.getValue(), TimeUtil.nowTs()); c.setGranted(timestamp); ins.add(c); addLabelDelta(normName, c.getValue()); categories.put(normName, c.getValue()); update.putApproval(ent.getKey(), ent.getValue()); } } forceCallerAsReviewer(rsrc, current, ins, upd, del); db.get().patchSetApprovals().delete(del); db.get().patchSetApprovals().insert(ins); db.get().patchSetApprovals().update(upd); return !del.isEmpty() || !ins.isEmpty() || !upd.isEmpty();
4625 1242677983GerritCodeReview/gerritShawn Pearcef9ae78e51a15f66d865142252920f494c91b9c65 None TODO(dborowitz): Wrap fewer exceptions if/when we kill gwtorm. SATD_ADDED load() public ChangeNotes load() throws OrmException if (!loaded) { Repository repo; try { repo = repoManager.openRepository(change.getProject()); } catch (IOException e) { throw new OrmException(e); } try { load(repo); loaded = true; } catch (ConfigInvalidException | IOException e) { throw new OrmException(e); } finally { repo.close(); } } return this;
4623 1242677982GerritCodeReview/gerritShawn Pearce23dc4d1bdc7520f74eeb06e1c086707f6db24418 None TODO(dborowitz): Wrap fewer exceptions if/when we kill gwtorm. SATD_ADDED apply(PatchSetApproval) public Timestamp apply(PatchSetApproval input) return input.getGranted();
4622 1242677981GerritCodeReview/gerritShawn Pearce132194e80ab616ab9ce910db256c5d1c0a1e3f58 None Every comment appears in both side maps as a linked pair. It is only necessary to search one side to find a comment on either side of the editor pair. SATD_ADDED commentNav(CodeMirror, Direction) return new Runnable() { @Override public void run() { // Every comment appears in both side maps as a linked pair. // It is only necessary to search one side to find a comment // on either side of the editor pair. SortedMap map = map(src.side()); int line = src.hasActiveLine() ? src.getLineNumber(src.getActiveLine()) + 1 : 0; if (dir == Direction.NEXT) { map = map.tailMap(line + 1); if (map.isEmpty()) { return; } line = map.firstKey(); } else { map = map.headMap(line); if (map.isEmpty()) { return; } line = map.lastKey(); } CommentGroup g = map.get(line); if (g.getBoxCount() == 0) { g = g.getPeer(); } CodeMirror cm = g.getCm(); double y = cm.heightAtLine(g.getLine() - 1, "local"); cm.setCursor(LineCharacter.create(g.getLine() - 1)); cm.scrollToY(y - 0.5 * cm.getScrollbarV().getClientHeight()); cm.focus(); } }; Runnable commentNav(final CodeMirror src, final Direction dir)
4619 1242677978GerritCodeReview/gerritShawn Pearce60f71145c9d09204f8bdb888754912a7c6a1b97b First line workaround First line workaround CLASS_OR_METHOD_CHANGED collapse(int, int, boolean) void collapse(int start, int end, boolean attach) if (attach) { boolean isNew = lineWidget == null; Configuration cfg = Configuration.create().set("coverGutter", true).set("noHScroll", true); if (start == 0) { // First line workaround lineWidget = cm.addLineWidget(end + 1, getElement(), cfg.set("above", true)); } else { lineWidget = cm.addLineWidget(start - 1, getElement(), cfg); } if (isNew) { setVisible(false); lineWidget.onFirstRedraw(new Runnable() { @Override public void run() { int w = cm.getGutterElement().getOffsetWidth(); getElement().getStyle().setPaddingLeft(w, Unit.PX); setVisible(true); } }); } } textMarker = cm.markText(CodeMirror.pos(start, 0), CodeMirror.pos(end), Configuration.create().set("collapsed", true).set("inclusiveLeft", true).set("inclusiveRight", true)); int skipped = end - start + 1; if (skipped <= UP_DOWN_THRESHOLD) { addStyleName(style.noExpand()); } else { upArrow.setHTML(PatchUtil.M.expandBefore(NUM_ROWS_TO_EXPAND)); downArrow.setHTML(PatchUtil.M.expandAfter(NUM_ROWS_TO_EXPAND)); } skipNum.setText(Integer.toString(skipped));
4617 1242677977GerritCodeReview/gerritShawn Pearce04bd5ac86720659bd34f95d20b605cb6c8e7ce67 TODO: This is not optimal, but shouldn't be too costly in most cases. Maybe rewrite after done keeping track of diff chunk positions. TODO: This is not optimal, but shouldn't be too costly in most cases. Maybe rewrite after done keeping track of diff chunk positions. SATD_MOVED_FILE splitSkips(int, List) List splitSkips(int context, List skips) // TODO: This is not optimal, but shouldn't be too costly in most cases. // Maybe rewrite after done keeping track of diff chunk positions. for (CommentBox box : lineActiveBox.values()) { int boxLine = box.getCommentInfo().line(); boolean sideA = box.getCm().side() == DisplaySide.A; List temp = new ArrayList(skips.size() + 2); for (SkippedLine skip : skips) { int startLine = sideA ? skip.getStartA() : skip.getStartB(); int deltaBefore = boxLine - startLine; int deltaAfter = startLine + skip.getSize() - boxLine; if (deltaBefore < -context || deltaAfter < -context) { // Size guaranteed to be greater than 1 temp.add(skip); } else if (deltaBefore > context && deltaAfter > context) { SkippedLine before = new SkippedLine(skip.getStartA(), skip.getStartB(), skip.getSize() - deltaAfter - context); skip.incrementStart(deltaBefore + context); checkAndAddSkip(temp, before); checkAndAddSkip(temp, skip); } else if (deltaAfter > context) { skip.incrementStart(deltaBefore + context); checkAndAddSkip(temp, skip); } else if (deltaBefore > context) { skip.reduceSize(deltaAfter + context); checkAndAddSkip(temp, skip); } } if (temp.isEmpty()) { return temp; } skips = temp; } return skips;
4614 1242677902GerritCodeReview/gerritShawn Pearcedc264431f7604616c2237847df23d96836949460 Email the reviewers The user knows they added themselves, don't bother emailing them. Email the reviewers The user knows they added themselves, don't bother emailing them. CLASS_OR_METHOD_CHANGED postAdd(Change, List) private void postAdd(Change change, List added) throws OrmException, EmailException if (added.isEmpty()) { return; } // Execute hook for added reviewers // PatchSet patchSet = dbProvider.get().patchSets().get(change.currentPatchSetId()); for (PatchSetApproval psa : added) { Account account = accountCache.get(psa.getAccountId()).getAccount(); hooks.doReviewerAddedHook(change, account, patchSet, dbProvider.get()); } // Email the reviewers // // The user knows they added themselves, don't bother emailing them. List toMail = Lists.newArrayListWithCapacity(added.size()); for (PatchSetApproval psa : added) { if (!psa.getAccountId().equals(currentUser.getAccountId())) { toMail.add(psa.getAccountId()); } } if (!added.isEmpty()) { try { AddReviewerSender cm = addReviewerSenderFactory.create(change); cm.setFrom(currentUser.getAccountId()); cm.addReviewers(toMail); cm.send(); } catch (Exception err) { log.error("Cannot send email to new reviewers of change " + change.getId(), err); } }
4613 1242677976GerritCodeReview/gerritShawn Pearced020143837c55ceb74ea8d88d0c87e99e5f11ace None TODO canSubmit? SATD_ADDED onInitUI() protected void onInitUI() super.onInitUI(); addStyleName(Gerrit.RESOURCES.css().publishCommentsScreen()); approvalButtons = new ArrayList(); descBlock = new ChangeDescriptionBlock(null); add(descBlock); approvals = new ApprovalTable(); add(approvals); final FormPanel form = new FormPanel(); final FlowPanel body = new FlowPanel(); form.setWidget(body); form.addSubmitHandler(new FormPanel.SubmitHandler() { @Override public void onSubmit(final SubmitEvent event) { event.cancel(); } }); add(form); approvalPanel = new FlowPanel(); body.add(approvalPanel); initMessage(body); draftsPanel = new FlowPanel(); body.add(draftsPanel); final FlowPanel buttonRow = new FlowPanel(); buttonRow.setStyleName(Gerrit.RESOURCES.css().patchSetActions()); body.add(buttonRow); send = new Button(Util.C.buttonPublishCommentsSend()); send.addClickHandler(this); buttonRow.add(send); submit = new Button(Util.C.buttonPublishSubmitSend()); submit.addClickHandler(this); buttonRow.add(submit); cancel = new Button(Util.C.buttonPublishCommentsCancel()); cancel.addClickHandler(this); buttonRow.add(cancel);
4611 1242677975GerritCodeReview/gerritShawn Pearce0fd8994e22dfb33b7696cb63bdd117b7513af2ba None Only execute this block once SATD_ADDED onDone(boolean) private void onDone(boolean enabled) setTabEnabled(tabInfo, enabled); outstandingCallbacks--; if (outstandingCallbacks == 0 || (enabled && tabInfo == Tab.RELATED_CHANGES)) { // Only execute this block once outstandingCallbacks = 0; for (int i = 0; i < getTabBar().getTabCount(); i++) { if (getTabBar().isTabEnabled(i)) { selectTab(i); setVisible(true); break; } } }
4610 1242677901GerritCodeReview/gerritShawn Pearcefaef658f386123f357fdaa2a32239700ed164e3f Does not account for draft status as a user might want to let a reviewer see a draft. Does not account for draft status as a user might want to let a reviewer see a draft. CLASS_OR_METHOD_CHANGED putGroup(ChangeResource, AddReviewerInput) private PostResult putGroup(ChangeResource rsrc, AddReviewerInput input) throws BadRequestException, UnprocessableEntityException, OrmException, EmailException, IOException GroupDescription.Basic group = groupsCollection.get().parseInternal(input.reviewer); PostResult result = new PostResult(); if (!isLegalReviewerGroup(group.getGroupUUID())) { result.error = MessageFormat.format(ChangeMessages.get().groupIsNotAllowed, group.getName()); return result; } Set reviewers = Sets.newLinkedHashSet(); ChangeControl control = rsrc.getControl(); Set members; try { members = groupMembersFactory.create(control.getCurrentUser()).listAccounts(group.getGroupUUID(), control.getProject().getNameKey()); } catch (NoSuchGroupException e) { throw new UnprocessableEntityException(e.getMessage()); } catch (NoSuchProjectException e) { throw new BadRequestException(e.getMessage()); } // if maxAllowed is set to 0, it is allowed to add any number of // reviewers int maxAllowed = cfg.getInt("addreviewer", "maxAllowed", DEFAULT_MAX_REVIEWERS); if (maxAllowed > 0 && members.size() > maxAllowed) { result.error = MessageFormat.format(ChangeMessages.get().groupHasTooManyMembers, group.getName()); return result; } // if maxWithoutCheck is set to 0, we never ask for confirmation int maxWithoutConfirmation = cfg.getInt("addreviewer", "maxWithoutConfirmation", DEFAULT_MAX_REVIEWERS_WITHOUT_CHECK); if (!input.confirmed() && maxWithoutConfirmation > 0 && members.size() > maxWithoutConfirmation) { result.confirm = true; result.error = MessageFormat.format(ChangeMessages.get().groupManyMembersConfirmation, group.getName(), members.size()); return result; } for (Account member : members) { if (member.isActive()) { IdentifiedUser user = identifiedUserFactory.create(member.getId()); // Does not account for draft status as a user might want to let a // reviewer see a draft. if (control.forUser(user).isRefVisible()) { reviewers.add(user); } } } addReviewers(rsrc, result, reviewers); return result;
4609 1242677974GerritCodeReview/gerritShawn Pearce66b8e6ccc885f97892e71e1edd6c9f23428e7471 None TODO: This is not optimal, but shouldn't be too costly in most cases. Maybe rewrite after done keeping track of diff chunk positions. SATD_ADDED renderSkips(int) private void renderSkips(int context) clearSkipBars(); if (context == AccountDiffPreference.WHOLE_FILE_CONTEXT) { return; } JsArray regions = diff.content(); List skips = new ArrayList(); int lineA = 0, lineB = 0; for (int i = 0; i < regions.length(); i++) { Region current = regions.get(i); if (current.ab() != null) { int len = current.ab().length(); if (i == 0 && len > context + 1) { skips.add(new SkippedLine(0, 0, len - context)); } else if (i == regions.length() - 1 && len > context + 1) { skips.add(new SkippedLine(lineA + context, lineB + context, len - context)); } else if (len > 2 * context + 1) { skips.add(new SkippedLine(lineA + context, lineB + context, len - 2 * context)); } lineA += len; lineB += len; } else { lineA += current.a() != null ? current.a().length() : 0; lineB += current.b() != null ? current.b().length() : 0; } } // TODO: This is not optimal, but shouldn't be too costly in most cases. // Maybe rewrite after done keeping track of diff chunk positions. skipBars = new HashSet(); for (CommentBox box : lineActiveBoxMap.values()) { List temp = new ArrayList(); for (SkippedLine skip : skips) { CommentInfo info = box.getCommentInfo(); int startLine = box.getSide() == DisplaySide.A ? skip.getStartA() : skip.getStartB(); int boxLine = info.line(); int deltaBefore = boxLine - startLine; int deltaAfter = startLine + skip.getSize() - boxLine; if (deltaBefore < -context || deltaAfter < -context) { // Size guaranteed to be greater than 1 temp.add(skip); } else if (deltaBefore > context && deltaAfter > context) { SkippedLine before = new SkippedLine(skip.getStartA(), skip.getStartB(), skip.getSize() - deltaAfter - context); skip.incrementStart(deltaBefore + context); checkAndAddSkip(temp, before); checkAndAddSkip(temp, skip); } else if (deltaAfter > context) { skip.incrementStart(deltaBefore + context); checkAndAddSkip(temp, skip); } else if (deltaBefore > context) { skip.reduceSize(deltaAfter + context); checkAndAddSkip(temp, skip); } } if (temp.isEmpty()) { return; } skips = temp; } for (SkippedLine skip : skips) { SkipBar barA = renderSkipHelper(cmA, skip); SkipBar barB = renderSkipHelper(cmB, skip); SkipBar.link(barA, barB); skipBars.add(barA); skipBars.add(barB); }
4607 1242677973GerritCodeReview/gerritDavid Pursehouse59b2ea066d07ef923e182022e9ed590ac93ed69f None TODO(sop): Switch to Dispatcher.toPatchSideBySide. SATD_ADDED url(PatchSet.Id, CommentInfo) private static String url(PatchSet.Id ps, CommentInfo info) // TODO(sop): Switch to Dispatcher.toPatchSideBySide. Change.Id c = ps.getParentKey(); return new StringBuilder().append("/c/").append(c.get()).append('/').append(ps.get()).append('/').append(KeyUtil.encode(info.path())).append(",cm").toString();
4604 1242677972GerritCodeReview/gerritShawn Pearceeb924ff74a90bbeb37cd3856dae4e9325598656b None TODO(sop): Switch to Dispatcher.toPatchSideBySide. SATD_ADDED url(PatchSet.Id, CommentInfo) private static String url(PatchSet.Id ps, CommentInfo info) // TODO(sop): Switch to Dispatcher.toPatchSideBySide. Change.Id c = ps.getParentKey(); return new StringBuilder().append("/c/").append(c.get()).append('/').append(ps.get()).append('/').append(KeyUtil.encode(info.path())).append(",cm").toString();
4603 1242677971GerritCodeReview/gerritDavid Pursehousebbf93f85cf4ce13cbd1a0fb030a61b966b801708 defer attribute here is to workaround IE running immediately. defer attribute here is to workaround IE running immediately. SATD_CHANGED inject(Document) public void inject(Document dom) String moduleName = selector.getModuleName(); StringBuilder s = new StringBuilder(); s.append("\n"); s.append("function ").append(moduleName).append("(){"); s.append("var s,l,t"); s.append(",w=window"); s.append(",d=document"); s.append(",n='").append(moduleName).append("'"); s.append(",f=d.createElement('iframe')"); s.append(";"); // Callback to execute the module once both s and l are true. // s.append("function m(){"); s.append("if(s&&l){"); // Base path needs to be absolute. There isn't an easy way to do this // other than forcing an image to load and then pulling the URL back. // s.append("var b,i=d.createElement('img');"); s.append("i.src=n+'/clear.cache.gif';"); s.append("b=i.src;"); s.append("b=b.substring(0,b.lastIndexOf('/')+1);"); // allow us to GC s.append(moduleName).append("=null;"); s.append("f.contentWindow.gwtOnLoad(undefined,n,b);"); s.append("}"); s.append("}"); // Set s true when the module script has finished loading. The // exact name here is known to the IFrameLinker and is called by // the code in the iframe. // s.append(moduleName).append(".onScriptLoad=function(){"); s.append("s=1;m();"); s.append("};"); // Set l true when the browser has finished processing the iframe // tag, and everything else on the page. // s.append(moduleName).append(".r=function(){"); s.append("l=1;m();"); s.append("};"); // Prevents mixed mode security in IE6/7. s.append("f.src=\"javascript:''\";"); s.append("f.id=n;"); s.append("f.style.cssText" + "='position:absolute;width:0;height:0;border:none';"); s.append("f.tabIndex=-1;"); s.append("d.body.appendChild(f);"); // The src has to be set after the iframe is attached to the DOM to avoid // refresh quirks in Safari. We have to use the location.replace trick to // avoid FF2 refresh quirks. // s.append("f.contentWindow.location.replace(n+'/").append(cacheHTML).append("');"); // defer attribute here is to workaround IE running immediately. // s.append("d.write('